diff --git a/Cargo.lock b/Cargo.lock index ab7d62a2a..0806e51fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -587,6 +587,7 @@ dependencies = [ "criterion", "encoding_rs", "evtx", + "flate2", "font-kit", "getrandom 0.3.4", "glob", @@ -597,8 +598,10 @@ dependencies = [ "quick-xml 0.41.0", "rayon", "regex", + "rusqlite", "serde", "serde_json", + "serde_norway", "sha2 0.11.0", "tauri", "tauri-build", @@ -628,6 +631,7 @@ version = "0.1.1" dependencies = [ "base64 0.22.1", "chrono", + "criterion", "encoding_rs", "log", "regex", @@ -1426,6 +1430,18 @@ dependencies = [ "zmij", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.4.1" @@ -2011,11 +2027,32 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + [[package]] name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32069d97bb81e38fa67eab65e3393bf804bb85969f2bc06bf13f64aef5aba248" +dependencies = [ + "hashbrown 0.17.1", +] [[package]] name = "heck" @@ -2711,6 +2748,17 @@ dependencies = [ "redox_syscall 0.7.4", ] +[[package]] +name = "libsqlite3-sys" +version = "0.38.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -3869,6 +3917,31 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.19", +] + +[[package]] +name = "rusqlite" +version = "0.40.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +dependencies = [ + "bitflags 2.11.1", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + [[package]] name = "rustc-hash" version = "2.1.2" @@ -3977,6 +4050,12 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "same-file" version = "1.0.6" @@ -4191,6 +4270,19 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_norway" +version = "0.9.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e408f29489b5fd500fab51ff1484fc859bb655f32c671f307dcd733b72e8168c" +dependencies = [ + "indexmap 2.11.4", + "itoa", + "ryu", + "serde", + "unsafe-libyaml-norway", +] + [[package]] name = "serde_repr" version = "0.1.20" @@ -4470,6 +4562,18 @@ dependencies = [ "system-deps", ] +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -5531,6 +5635,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "unsafe-libyaml-norway" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39abd59bf32521c7f2301b52d05a6a2c975b6003521cbd0c6dc1582f0a22104" + [[package]] name = "untrusted" version = "0.9.0" @@ -5642,6 +5752,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version-compare" version = "0.2.1" diff --git a/crates/cmtraceopen-parser/Cargo.toml b/crates/cmtraceopen-parser/Cargo.toml index 1acf9c263..17126e925 100644 --- a/crates/cmtraceopen-parser/Cargo.toml +++ b/crates/cmtraceopen-parser/Cargo.toml @@ -29,3 +29,9 @@ base64 = "0.22" sha2 = "0.11" [dev-dependencies] +# Benchmarks only. A dev-dependency, so it never reaches the wasm32 build of the library. +criterion = "0.8" + +[[bench]] +name = "eventmap_apply" +harness = false diff --git a/crates/cmtraceopen-parser/benches/eventmap_apply.rs b/crates/cmtraceopen-parser/benches/eventmap_apply.rs new file mode 100644 index 000000000..0968819d3 --- /dev/null +++ b/crates/cmtraceopen-parser/benches/eventmap_apply.rs @@ -0,0 +1,87 @@ +//! How long applying one map to one event takes. +//! +//! The map engine runs once per record, so this is multiplied by the channel size: a million-record +//! scan pays it a million times. The benchmark exists because the path expressions and `%Name%` +//! placeholders in a map are constant for its whole life, and the applier used to rebuild both for +//! every record. +//! +//! Measured on an M-series mac against the Security 4624 fixture, comparing re-parsing per record +//! against the compiled cache: +//! +//! | | per record | per 10k records | +//! |---|---|---| +//! | re-parsing each time | 5.20 us | 49.0 ms | +//! | compiled once | 1.91 us | 18.9 ms | +//! +//! Run with `cargo bench --bench eventmap_apply` from `crates/cmtraceopen-parser`. + +use criterion::{criterion_group, criterion_main, Criterion}; +use std::hint::black_box; + +use cmtraceopen_parser::eventmap::{apply_map, EventMap, EventNode}; + +/// A map shaped like the upstream Security 4624: several bindings, multi-placeholder templates. +const SECURITY_4624: &str = include_str!("../tests/fixtures/eventmap/security-4624.json"); + +fn event() -> EventNode { + EventNode::new("Event").with_child( + EventNode::new("EventData") + .with_child( + EventNode::new("Data") + .with_attribute("Name", "SubjectUserName") + .with_text("adam"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "SubjectDomainName") + .with_text("CONTOSO"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "TargetUserName") + .with_text("svc-backup"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "IpAddress") + .with_text("10.0.0.7"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "LogonType") + .with_text("10"), + ), + ) +} + +fn apply_one_record(c: &mut Criterion) { + let map: EventMap = serde_json::from_str(SECURITY_4624).expect("fixture parses"); + let event = event(); + + // Warm the compiled cache the way a real scan does: the first record pays for the parse, and + // every record after it reuses the result. Measuring from cold would report the one-off cost + // rather than the per-record cost that actually multiplies. + let _ = apply_map(&map, &event); + + c.bench_function("apply_map/security-4624/one record", |b| { + b.iter(|| black_box(apply_map(black_box(&map), black_box(&event)))); + }); +} + +fn apply_a_channel(c: &mut Criterion) { + let map: EventMap = serde_json::from_str(SECURITY_4624).expect("fixture parses"); + let event = event(); + let _ = apply_map(&map, &event); + + // Ten thousand records is a small channel; the point is the shape of the curve, not the total. + c.bench_function("apply_map/security-4624/10k records", |b| { + b.iter(|| { + for _ in 0..10_000 { + black_box(apply_map(black_box(&map), black_box(&event))); + } + }); + }); +} + +criterion_group!(benches, apply_one_record, apply_a_channel); +criterion_main!(benches); diff --git a/crates/cmtraceopen-parser/src/event_payload/mod.rs b/crates/cmtraceopen-parser/src/event_payload/mod.rs new file mode 100644 index 000000000..c9a84baa8 --- /dev/null +++ b/crates/cmtraceopen-parser/src/event_payload/mod.rs @@ -0,0 +1,309 @@ +//! Decoding the `EventPayload` element found in `.etl` traces. +//! +//! Some providers, notably the Windows Update trace under `C:\Windows\Logs\WindowsUpdate`, do not +//! write structured `EventData`. They write a single `EventPayload` element holding the message as +//! a hexadecimal string. Rendered as-is it is a wall of hex digits, which is why those traces are +//! usually described as unreadable without a dedicated tool. +//! +//! FullEventLogView added this conversion in 1.55 and it is the reason those logs are readable +//! there at all. This is the same idea, implemented as a pure function. +//! +//! Encoding is decided by inspection rather than assumption. Windows writes these payloads as +//! UTF-16LE, but not universally, and guessing wrong turns readable text into interleaved nulls or +//! mojibake. Both interpretations are scored and the better one wins, with a refusal when neither +//! is convincing, in which case the caller shows the hex unchanged. + +use crate::eventmap::EventNode; + +/// How the payload bytes were interpreted. +// Growable: UTF-16BE and single-byte code pages are both plausible additions. Marking it now keeps +// adding one a minor change; after the first release that exposes the type it is itself breaking. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PayloadEncoding { + /// UTF-16 little endian, which is what Windows writes most often. + Utf16Le, + /// Single-byte text, either ASCII or UTF-8. + Utf8, +} + +/// A decoded payload. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DecodedPayload { + /// The decoded message, with any trailing NUL padding already removed. + /// + /// Never empty: a payload that decodes to nothing is refused rather than returned, since empty + /// text would read as an event that said nothing rather than one that was not understood. + pub text: String, + /// Which reading of the bytes produced [`text`](Self::text). + /// + /// Reported rather than hidden because it is a decision made by inspection, and an operator + /// looking at a suspicious message needs to know the encoding was inferred. + pub encoding: PayloadEncoding, +} + +/// Parses a hexadecimal string into bytes. +/// +/// Whitespace is ignored, since providers sometimes wrap long payloads. An odd digit count or a +/// non-hex character means this is not a hex payload at all, which is a refusal rather than a best +/// effort: decoding half a string would present truncated evidence as complete. +fn hex_to_bytes(hex: &str) -> Option> { + let digits: Vec = hex + .bytes() + .filter(|byte| !byte.is_ascii_whitespace()) + .collect(); + if digits.is_empty() || !digits.len().is_multiple_of(2) { + return None; + } + + let mut bytes = Vec::with_capacity(digits.len() / 2); + for pair in digits.chunks_exact(2) { + let high = (pair[0] as char).to_digit(16)?; + let low = (pair[1] as char).to_digit(16)?; + bytes.push(((high << 4) | low) as u8); + } + Some(bytes) +} + +/// Fraction of characters that are ASCII printable or ordinary whitespace. +/// +/// Deliberately stricter than "not a control character". Misreading arbitrary bytes as UTF-16 +/// produces letter-like characters from the Latin Extended and CJK ranges, which are not control +/// characters and would sail through a looser check. Requiring ASCII rejects that, because the +/// signature of a wrong interpretation is text that is suddenly not ASCII at all. +/// +/// The cost is that a payload written in a non-Latin script falls back to raw hex rather than +/// being decoded. That is the conservative direction: raw hex is visibly unreadable, whereas +/// mojibake looks like data and misleads. Windows writes these traces in English. +fn readable_ratio(text: &str) -> f32 { + let total = text.chars().count(); + if total == 0 { + return 0.0; + } + let readable = text + .chars() + .filter(|c| matches!(c, ' '..='~' | '\n' | '\r' | '\t')) + .count(); + readable as f32 / total as f32 +} + +fn decode_utf16le(bytes: &[u8]) -> Option { + if !bytes.len().is_multiple_of(2) { + return None; + } + let units: Vec = bytes + .chunks_exact(2) + .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) + .collect(); + String::from_utf16(&units).ok() +} + +/// Text is accepted only when this much of it is readable. +/// +/// Set high because the alternative to accepting is showing the raw hex, which is at least +/// obviously unreadable. Mojibake looks like data and misleads. +const MINIMUM_READABLE_RATIO: f32 = 0.9; + +/// Decodes an `EventPayload` hex string into readable text. +/// +/// Returns `None` when the input is not hexadecimal, or when neither interpretation produces +/// convincingly readable text. The caller should then show the payload unchanged. +pub fn decode_event_payload(hex: &str) -> Option { + let bytes = hex_to_bytes(hex.trim())?; + + let mut best: Option<(String, f32, PayloadEncoding)> = None; + + if let Some(text) = decode_utf16le(&bytes) { + let trimmed = text.trim_end_matches('\0').to_string(); + let ratio = readable_ratio(&trimmed); + best = Some((trimmed, ratio, PayloadEncoding::Utf16Le)); + } + + if let Ok(text) = String::from_utf8(bytes) { + let trimmed = text.trim_end_matches('\0').to_string(); + let ratio = readable_ratio(&trimmed); + // Strictly greater, so ties go to UTF-16: Windows writes these payloads that way far more + // often, and a short ASCII payload decodes plausibly under both readings. + let better = match &best { + Some((_, best_ratio, _)) => ratio > *best_ratio, + None => true, + }; + if better { + best = Some((trimmed, ratio, PayloadEncoding::Utf8)); + } + } + + let (text, ratio, encoding) = best?; + if text.is_empty() || ratio < MINIMUM_READABLE_RATIO { + return None; + } + + Some(DecodedPayload { text, encoding }) +} + +/// The element name providers use for a hex-encoded message body. +const PAYLOAD_ELEMENT: &str = "EventPayload"; + +/// Finds and decodes the `EventPayload` element anywhere in a parsed event. +/// +/// The element's position varies: some providers put it under `UserData`, others under +/// `EventData`, and the wrapper element carries the provider's own name. Searching the whole tree +/// avoids hard-coding a path that would silently match nothing for half of them. +/// +/// The first decodable payload wins. Events carrying more than one are not something Windows +/// emits, and picking the first is at least deterministic. +pub fn decode_payload_in(root: &EventNode) -> Option { + if root.name == PAYLOAD_ELEMENT { + if let Some(decoded) = root.text.as_deref().and_then(decode_event_payload) { + return Some(decoded); + } + } + root.children.iter().find_map(decode_payload_in) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn utf16_hex(text: &str) -> String { + text.encode_utf16() + .flat_map(|unit| unit.to_le_bytes()) + .map(|byte| format!("{byte:02X}")) + .collect() + } + + fn utf8_hex(text: &str) -> String { + text.bytes().map(|byte| format!("{byte:02X}")).collect() + } + + #[test] + fn decodes_a_utf16le_payload() { + let decoded = + decode_event_payload(&utf16_hex("Agent * START * Finding updates")).expect("decodes"); + assert_eq!(decoded.text, "Agent * START * Finding updates"); + assert_eq!(decoded.encoding, PayloadEncoding::Utf16Le); + } + + #[test] + fn decodes_a_single_byte_payload() { + // Long enough that the UTF-16 reading is clearly worse, which is what decides it. + let text = "Downloading update 8f21b4c2 from the delivery service"; + let decoded = decode_event_payload(&utf8_hex(text)).expect("decodes"); + assert_eq!(decoded.text, text); + assert_eq!(decoded.encoding, PayloadEncoding::Utf8); + } + + #[test] + fn tolerates_whitespace_between_digits() { + assert_eq!( + decode_event_payload("48 00 69 00").expect("decodes").text, + "Hi" + ); + } + + #[test] + fn accepts_lowercase_hex() { + assert_eq!( + decode_event_payload(&utf16_hex("Hi").to_lowercase()) + .expect("decodes") + .text, + "Hi" + ); + } + + #[test] + fn strips_a_trailing_null_terminator() { + assert_eq!( + decode_event_payload(&utf16_hex("Done\0")) + .expect("decodes") + .text, + "Done" + ); + } + + #[test] + fn preserves_embedded_newlines() { + assert_eq!( + decode_event_payload(&utf16_hex("line one\r\nline two")) + .expect("decodes") + .text, + "line one\r\nline two" + ); + } + + #[test] + fn refuses_an_odd_number_of_digits() { + // Decoding half of it would present truncated evidence as complete. + assert!(decode_event_payload("48656C6C6").is_none()); + } + + #[test] + fn refuses_input_that_is_not_hexadecimal() { + assert!(decode_event_payload("not hex at all").is_none()); + assert!(decode_event_payload("ZZ00").is_none()); + assert!(decode_event_payload("").is_none()); + } + + #[test] + fn refuses_binary_that_is_not_text() { + // Both readings "succeed" here and neither is text: as UTF-8 these are control bytes, and + // as UTF-16 they become Latin Extended letters that look like language but are not. Showing + // either as a message would be a confident wrong answer; raw hex is at least honest. + let binary: String = (0u8..64).map(|byte| format!("{byte:02X}")).collect(); + assert!(decode_event_payload(&binary).is_none()); + } + + #[test] + fn refuses_a_payload_that_decodes_to_nothing() { + assert!(decode_event_payload("0000").is_none()); + } + + #[test] + fn a_short_ascii_payload_is_read_as_utf16_when_both_are_plausible() { + let decoded = decode_event_payload(&utf16_hex("OK")).expect("decodes"); + assert_eq!(decoded.text, "OK"); + assert_eq!(decoded.encoding, PayloadEncoding::Utf16Le); + } + + #[test] + fn finds_a_payload_nested_under_a_provider_wrapper() { + // The wrapper element carries the provider's own name, so the path cannot be hard-coded. + let root = EventNode::new("Event").with_child( + EventNode::new("UserData") + .with_child(EventNode::new("WindowsUpdateClient").with_child( + EventNode::new("EventPayload").with_text(utf16_hex("Agent ready")), + )), + ); + assert_eq!(decode_payload_in(&root).expect("found").text, "Agent ready"); + } + + #[test] + fn an_event_without_a_payload_yields_nothing() { + let root = EventNode::new("Event").with_child( + EventNode::new("EventData") + .with_child(EventNode::new("Data").with_text(utf16_hex("not a payload"))), + ); + assert!(decode_payload_in(&root).is_none()); + } + + #[test] + fn an_undecodable_payload_does_not_hide_a_later_decodable_one() { + // Refusing the first must not end the search, or one malformed element would suppress the + // readable message that follows it. + let root = EventNode::new("Event") + .with_child(EventNode::new("EventPayload").with_text("ZZZZ")) + .with_child(EventNode::new("EventPayload").with_text(utf16_hex("readable"))); + assert_eq!(decode_payload_in(&root).expect("found").text, "readable"); + } + + #[test] + fn round_trips_a_realistic_windows_update_line() { + let line = "2026-08-09 12:00:00.1234567 1234 5678 Agent Update service is being installed"; + assert_eq!( + decode_event_payload(&utf16_hex(line)) + .expect("decodes") + .text, + line + ); + } +} diff --git a/crates/cmtraceopen-parser/src/event_query/mod.rs b/crates/cmtraceopen-parser/src/event_query/mod.rs new file mode 100644 index 000000000..1d7f14139 --- /dev/null +++ b/crates/cmtraceopen-parser/src/event_query/mod.rs @@ -0,0 +1,1605 @@ +//! Building XPath queries for the Windows Event Log service. +//! +//! Filtering can happen in two places: inside the service, or in the client after every matching +//! event has been fetched and rendered. The difference is not marginal. FullEventLogView pushes +//! only Event ID down and evaluates level, provider, time, and description client-side, which is +//! why its default "last 7 days" costs a full walk of every channel. Reverse iteration plus an +//! early exit is what makes that survivable, not the filter itself. +//! +//! This module builds the query so the service does the work. It is pure string construction with +//! no Windows dependency, which keeps it inside the parser crate and unit-testable off Windows. +//! +//! Four constraints come from the service rather than from taste: +//! +//! - **Expression count is capped.** A query with too many `or` terms is rejected outright, so +//! large Event ID sets are split across several `` node. +/// +/// Counting *selectors* rather than expressions gets this wrong: a range emits two comparisons, so +/// ten ranges already exhaust the budget before any level, provider, or time term is added. The +/// budget is therefore spent in expressions. +/// +/// The limit is measured, not taken from the documentation. Microsoft documents 32 expressions per +/// XPath, but the service on Windows 11 build 26200 accepts 23 and rejects 24 with +/// `ERROR_EVT_INVALID_QUERY` (15001), so the documented figure would have produced queries that +/// fail outright. Twenty leaves three expressions of headroom for a miscount, which is exactly what +/// absorbed the two-bounded time window being costed as one expression rather than two. +const MAX_EXPRESSIONS_PER_SELECT: usize = 20; + +/// The expression count at which the service starts refusing a query. +/// +/// Measured on Windows 11 build 26200 against `Application` with strict flags: 23 `or`-joined +/// comparisons are accepted, 24 and every count tried up to 50 are rejected with +/// `ERROR_EVT_INVALID_QUERY` (15001). Recorded as a constant so the budget above is visibly +/// derived from a measurement rather than from the documented figure of 32, which is wrong here. +const MEASURED_REJECTION_POINT: usize = 24; + +// The budget must leave room, not merely differ. Checked at compile time so raising it past what +// the service accepts cannot reach a user. +const _: () = assert!(MAX_EXPRESSIONS_PER_SELECT < MEASURED_REJECTION_POINT); + +/// Quotes a value as an XPath string literal, choosing a delimiter the value does not contain. +/// +/// XPath 1.0 has no escape for either delimiter but accepts both, so a value containing one can be +/// quoted with the other. Deleting the apostrophe instead would silently change the value: a +/// provider named `Bob's Agent` would become `Bobs Agent` and match nothing, turning a filter into +/// a silent no-op rather than a visible error. +/// +/// A value containing both delimiters cannot be expressed at all, so it is refused rather than +/// mangled. XML metacharacters are deliberately not touched here; whether they need escaping +/// depends on where the expression lands, which is [`escape_for_xml`]'s job. +fn quote_literal(value: &str) -> Result { + if !value.contains('\'') { + Ok(format!("'{value}'")) + } else if !value.contains('"') { + Ok(format!("\"{value}\"")) + } else { + Err(QueryBuildError::UnquotableValue(value.to_string())) + } +} + +/// Why a filter could not be compiled into a query. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +#[non_exhaustive] +pub enum QueryBuildError { + /// The value contains both `'` and `"`, which an XPath 1.0 string literal cannot express. + #[error("value contains both quote characters and cannot be an XPath string literal: {0}")] + UnquotableValue(String), + /// The terms that cannot be split across nodes already exceed what one node may carry. + /// + /// Levels, providers, the time window and the keyword mask repeat in every node, so no amount + /// of chunking the Event IDs can bring them under the limit. Refused rather than emitted, + /// because the service rejects an oversized query and the tolerate-errors flag turns that + /// refusal into a channel that reports no events at all. + #[error( + "filter needs {needed} expressions before Event IDs, more than the {limit} one query node \ + may carry; narrow the levels or providers" + )] + FilterTooComplex { + /// Expressions the unsplittable terms require. + needed: usize, + /// Expressions one node may carry. + limit: usize, + }, +} + +/// XML-escapes a complete XPath expression for embedding inside a `` document. +fn escape_for_xml(expression: &str) -> String { + let mut escaped = String::with_capacity(expression.len()); + for character in expression.chars() { + match character { + '&' => escaped.push_str("&"), + '<' => escaped.push_str("<"), + '>' => escaped.push_str(">"), + '"' => escaped.push_str("""), + _ => escaped.push(character), + } + } + escaped +} + +fn join_or(predicates: &[String]) -> String { + format!("({})", predicates.join(" or ")) +} + +fn join_and(predicates: &[String]) -> String { + format!("({})", predicates.join(" and ")) +} + +/// A predicate and the number of expressions the service will count it as. +/// +/// Returned together on purpose. The cost was previously computed by a separate function that +/// assumed one expression per predicate, which is wrong for a two-bounded window: it emits two +/// comparisons joined by `and`. Deriving both from the same code is what stops them drifting +/// apart again, and drift here is expensive because the result is a query the service rejects +/// outright rather than a slightly wrong one. +struct Predicate { + clause: String, + expressions: usize, +} + +fn time_predicate(window: &TimeWindow) -> Result, QueryBuildError> { + match window { + TimeWindow::Last { milliseconds } => Ok(Some(Predicate { + clause: format!("TimeCreated[timediff(@SystemTime) <= {milliseconds}]"), + expressions: 1, + })), + TimeWindow::Between { from, to } => { + let mut bounds = Vec::new(); + if let Some(from) = from { + bounds.push(format!("@SystemTime >= {}", quote_literal(from)?)); + } + if let Some(to) = to { + bounds.push(format!("@SystemTime <= {}", quote_literal(to)?)); + } + if bounds.is_empty() { + return Ok(None); + } + Ok(Some(Predicate { + expressions: bounds.len(), + clause: format!("TimeCreated[{}]", bounds.join(" and ")), + })) + } + } +} + +/// Levels with duplicates removed, preserving the order they were given in. +/// +/// A caller can legitimately hand the same level twice. Emitting `Level=2 or Level=2` costs two +/// expressions to say one thing, and the budget is small enough that spending it that way pushes a +/// query over the limit for no benefit. +fn distinct_levels(filter: &EventQueryFilter) -> Vec { + let mut seen = Vec::new(); + for level in &filter.levels { + if !seen.contains(level) { + seen.push(*level); + } + } + seen +} + +/// Provider names with duplicates removed, compared case-insensitively as the service matches. +fn distinct_providers(filter: &EventQueryFilter) -> Vec { + let mut seen: Vec = Vec::new(); + for provider in &filter.providers { + if !seen + .iter() + .any(|existing| existing.eq_ignore_ascii_case(provider)) + { + seen.push(provider.clone()); + } + } + seen +} + +/// Expressions contributed by everything other than the Event ID list. +/// +/// These repeat in every node, so they are what decides whether chunking the Event IDs can bring a +/// query under the limit at all. +/// The most expressions any single Event ID selector costs. +/// +/// A selector cannot be split: a range is two comparisons and stays together. So if the budget +/// left after the fixed terms is smaller than this, no chunking can produce a conforming node. +fn largest_selector_cost(filter: &EventQueryFilter) -> usize { + filter + .event_ids + .iter() + .map(EventIdSelector::expression_cost) + .max() + .unwrap_or(0) +} + +fn fixed_expression_cost(filter: &EventQueryFilter) -> usize { + let time = filter + .time + .as_ref() + .and_then(|window| time_predicate(window).ok().flatten()) + .map(|predicate| predicate.expressions) + .unwrap_or(0); + time + distinct_levels(filter).len() + + distinct_providers(filter).len() + + usize::from(filter.keywords.is_some()) +} + +fn system_predicates( + filter: &EventQueryFilter, + event_ids: &[EventIdSelector], +) -> Result, QueryBuildError> { + let mut predicates = Vec::new(); + + if let Some(window) = filter.time.as_ref() { + if let Some(predicate) = time_predicate(window)? { + predicates.push(predicate.clause); + } + } + + let levels = distinct_levels(filter); + if !levels.is_empty() { + let levels: Vec = levels + .iter() + .map(|level| format!("Level={level}")) + .collect(); + predicates.push(join_or(&levels)); + } + + if !event_ids.is_empty() { + let ids: Vec = event_ids + .iter() + .map(|selector| selector.predicate(filter.event_id_mode)) + .collect(); + // Include is a union of alternatives; exclude must hold for every listed id at once. + predicates.push(match filter.event_id_mode { + SelectorMode::Include => join_or(&ids), + SelectorMode::Exclude => join_and(&ids), + }); + } + + let distinct = distinct_providers(filter); + if !distinct.is_empty() { + let (operator, joiner) = match filter.provider_mode { + SelectorMode::Include => ("=", " or "), + SelectorMode::Exclude => ("!=", " and "), + }; + let mut providers = Vec::with_capacity(distinct.len()); + for provider in &distinct { + providers.push(format!("@Name{operator}{}", quote_literal(provider)?)); + } + predicates.push(format!("Provider[{}]", providers.join(joiner))); + } + + if let Some(keywords) = filter.keywords { + predicates.push(format!("band(Keywords,{keywords})")); + } + + Ok(predicates) +} + +fn select_body( + filter: &EventQueryFilter, + event_ids: &[EventIdSelector], +) -> Result { + let predicates = system_predicates(filter, event_ids)?; + if predicates.is_empty() { + return Ok("*".to_string()); + } + Ok(format!("*[System[{}]]", predicates.join(" and "))) +} + +/// Splits Event ID selectors so each group fits the expression budget alongside the fixed terms. +/// +/// Every returned chunk costs at most `MAX_EXPRESSIONS_PER_SELECT - fixed_cost`. The caller must +/// have already established that no single selector exceeds that, which +/// [`largest_selector_cost`] is for; there is no fallback that emits an oversized node. +fn chunk_by_expression_budget( + selectors: &[EventIdSelector], + fixed_cost: usize, +) -> Vec> { + let budget = MAX_EXPRESSIONS_PER_SELECT.saturating_sub(fixed_cost); + let mut chunks: Vec> = Vec::new(); + let mut current: Vec = Vec::new(); + let mut spent = 0usize; + + for selector in selectors { + let cost = selector.expression_cost(); + if !current.is_empty() && spent + cost > budget { + chunks.push(std::mem::take(&mut current)); + spent = 0; + } + current.push(*selector); + spent += cost; + } + if !current.is_empty() { + chunks.push(current); + } + chunks +} + +/// Builds the query string passed to `EvtQuery`. +/// +/// Returns `*` when nothing is filtered. When the Event ID set does not fit one node's expression +/// budget, the result is a `` whose nodes each repeat the other predicates, because the +/// service unions the nodes rather than intersecting them. +pub fn build_query(filter: &EventQueryFilter) -> Result { + if filter.is_unfiltered() { + return Ok("*".to_string()); + } + + let fixed_cost = fixed_expression_cost(filter); + let event_id_cost: usize = filter + .event_ids + .iter() + .map(EventIdSelector::expression_cost) + .sum(); + + // Refused before anything is built. These terms repeat in every node, so no amount of chunking + // the Event IDs brings them under the limit; emitting anyway produces a query the service + // rejects, and the tolerate-errors flag turns that into a channel reporting no events. An + // error the caller can show beats a filter that appears to work and returns nothing. + // The unsplittable terms plus the largest single selector, since a selector cannot be divided: + // a range is two comparisons and stays together. Checking only the fixed terms left a gap where + // nineteen providers plus one range emitted a node of twenty-one expressions, because the + // chunker's old floor handed out a budget of one to a selector that costs two. + let indivisible = fixed_cost + largest_selector_cost(filter); + if indivisible > MAX_EXPRESSIONS_PER_SELECT { + return Err(QueryBuildError::FilterTooComplex { + needed: indivisible, + limit: MAX_EXPRESSIONS_PER_SELECT, + }); + } + + let over_budget = + !filter.event_ids.is_empty() && fixed_cost + event_id_cost > MAX_EXPRESSIONS_PER_SELECT; + + // An exclusion list cannot be split across unioned nodes: "not (a or b)" spread that + // way means "not a or not b", which matches nearly everything. It is expressed with + // instead, which the service subtracts from the selection rather than unioning, so chunking is + // safe. Measured on Windows 11 build 26200: a 30-term "!=" chain is rejected outright with + // ERROR_EVT_INVALID_QUERY, chunked is accepted, and on a list small enough for both + // forms the two return identical counts. + // + // This mattered more than a rejection normally would. Production sets + // EvtQueryTolerateQueryErrors, which turns that refusal into a channel that reports no events, + // so the filter appeared to work and silently returned nothing. + if over_budget && filter.event_id_mode == SelectorMode::Exclude { + return build_suppressed_query(filter); + } + + let needs_split = over_budget && filter.event_id_mode == SelectorMode::Include; + + if !needs_split { + return select_body(filter, &filter.event_ids); + } + + let mut query = String::from(""); + for (id, chunk) in chunk_by_expression_budget(&filter.event_ids, fixed_cost) + .iter() + .enumerate() + { + // The schema documents Id as required once the list holds more than one Query. The service + // does not enforce it: a two-node list without Id was measured returning exactly the same + // events as the equivalent single expression. It is written anyway because it costs + // nothing and the same document shape is what a saved custom view is validated against. + // + // No Path is written. EvtQuery supplies the channel from its own argument when the + // document omits it, and the schema requires that if any node names a path they all do, so + // omitting it everywhere is the consistent choice. + // + // The expression becomes XML text here, so it is escaped at exactly this boundary. + let _ = write!( + query, + "", + escape_for_xml(&select_body(filter, chunk)?) + ); + } + query.push_str(""); + Ok(query) +} + +/// Builds an exclusion too large for one node as `"); + query.push_str(&escape_for_xml(&selection)); + query.push_str(""); + + // Suppressions are written as inclusions of what to remove, so each selector costs what it + // would cost in an include list. + for chunk in chunk_by_expression_budget(&filter.event_ids, 0) { + let predicates: Vec = chunk + .iter() + .map(|selector| selector.predicate(SelectorMode::Include)) + .collect(); + let body = format!("*[System[({})]]", join_or(&predicates)); + let _ = write!(query, "{}", escape_for_xml(&body)); + } + + query.push_str(""); + Ok(query) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn filter() -> EventQueryFilter { + EventQueryFilter::default() + } + + #[test] + fn an_empty_filter_is_the_unfiltered_wildcard() { + assert!(filter().is_unfiltered()); + assert_eq!(build_query(&filter()).expect("builds"), "*"); + } + + #[test] + fn a_relative_window_uses_timediff_against_the_service_clock() { + let mut f = filter(); + f.time = Some(TimeWindow::Last { + milliseconds: 604_800_000, + }); + assert_eq!( + build_query(&f).expect("builds"), + "*[System[TimeCreated[timediff(@SystemTime) <= 604800000]]]" + ); + } + + #[test] + fn an_absolute_range_emits_both_bounds() { + let mut f = filter(); + f.time = Some(TimeWindow::Between { + from: Some("2026-08-01T00:00:00.000Z".into()), + to: Some("2026-08-09T00:00:00.000Z".into()), + }); + assert_eq!( + build_query(&f).expect("builds"), + "*[System[TimeCreated[@SystemTime >= '2026-08-01T00:00:00.000Z' and @SystemTime <= '2026-08-09T00:00:00.000Z']]]" + ); + } + + #[test] + fn a_half_open_range_emits_only_the_bound_that_was_given() { + let mut f = filter(); + f.time = Some(TimeWindow::Between { + from: Some("2026-08-01T00:00:00.000Z".into()), + to: None, + }); + assert_eq!( + build_query(&f).expect("builds"), + "*[System[TimeCreated[@SystemTime >= '2026-08-01T00:00:00.000Z']]]" + ); + } + + #[test] + fn a_range_with_no_bounds_contributes_nothing() { + let mut f = filter(); + f.time = Some(TimeWindow::Between { + from: None, + to: None, + }); + f.levels = vec![2]; + assert_eq!(build_query(&f).expect("builds"), "*[System[(Level=2)]]"); + } + + #[test] + fn levels_are_unioned() { + let mut f = filter(); + f.levels = vec![1, 2, 3]; + assert_eq!( + build_query(&f).expect("builds"), + "*[System[(Level=1 or Level=2 or Level=3)]]" + ); + } + + #[test] + fn event_ids_support_single_values_and_ranges() { + let mut f = filter(); + f.event_ids = vec![ + EventIdSelector::Single { id: 4624 }, + EventIdSelector::Range { + low: 5000, + high: 5010, + }, + ]; + assert_eq!( + build_query(&f).expect("builds"), + "*[System[(EventID=4624 or (EventID >= 5000 and EventID <= 5010))]]" + ); + } + + #[test] + fn a_reversed_range_is_normalized_rather_than_emitted_backwards() { + let mut f = filter(); + f.event_ids = vec![EventIdSelector::Range { low: 9, high: 5 }]; + assert_eq!( + build_query(&f).expect("builds"), + "*[System[((EventID >= 5 and EventID <= 9))]]" + ); + } + + #[test] + fn a_degenerate_range_collapses_to_a_single_id() { + let mut f = filter(); + f.event_ids = vec![EventIdSelector::Range { low: 7, high: 7 }]; + assert_eq!(build_query(&f).expect("builds"), "*[System[(EventID=7)]]"); + } + + #[test] + fn exclusion_negates_the_whole_clause() { + let mut f = filter(); + f.event_ids = vec![EventIdSelector::Single { id: 4688 }]; + f.event_id_mode = SelectorMode::Exclude; + assert_eq!( + build_query(&f).expect("builds"), + "*[System[(EventID!=4688)]]" + ); + } + + #[test] + fn providers_are_matched_by_name() { + let mut f = filter(); + f.providers = vec!["Microsoft-Windows-Shell-Core".into()]; + assert_eq!( + build_query(&f).expect("builds"), + "*[System[Provider[@Name='Microsoft-Windows-Shell-Core']]]" + ); + } + + #[test] + fn keywords_are_matched_with_band() { + let mut f = filter(); + f.keywords = Some(0x8000_0000_0000_0000); + assert_eq!( + build_query(&f).expect("builds"), + "*[System[band(Keywords,9223372036854775808)]]" + ); + } + + #[test] + fn multiple_dimensions_are_intersected() { + let mut f = filter(); + f.time = Some(TimeWindow::Last { + milliseconds: 3_600_000, + }); + f.levels = vec![2]; + f.event_ids = vec![EventIdSelector::Single { id: 1000 }]; + f.providers = vec!["ESENT".into()]; + assert_eq!( + build_query(&f).expect("builds"), + "*[System[TimeCreated[timediff(@SystemTime) <= 3600000] and (Level=2) and (EventID=1000) and Provider[@Name='ESENT']]]" + ); + } + + #[test] + fn a_large_id_set_is_split_across_select_nodes_in_one_query_list() { + let mut f = filter(); + f.event_ids = (1..=45).map(|id| EventIdSelector::Single { id }).collect(); + let query = build_query(&f).expect("builds"); + + assert!(query.starts_with("")); + assert!(query.ends_with("")); + assert_eq!( + query.matches("").count() >= 2); + assert_eq!( + query.matches("(Level=2)").count(), + query.matches("").count(), + 1, + "one selection: {query}" + ); + assert!( + query.matches("").count() >= 2, + "the list must be chunked: {query}" + ); + // Suppressions say what to remove, so they are written as equalities, not "!=". + assert!(query.contains("EventID=1 or EventID=2"), "{query}"); + assert!(!query.contains("EventID!="), "{query}"); + assert!(!query.contains("not("), "{query}"); + } + + #[test] + fn every_suppression_node_stays_within_the_budget() { + let mut f = filter(); + f.event_ids = (1..=200).map(|id| EventIdSelector::Single { id }).collect(); + f.event_id_mode = SelectorMode::Exclude; + let query = build_query(&f).expect("builds"); + + for node in query.split("").skip(1) { + let body = node.split("").next().unwrap_or_default(); + assert!( + body.matches("EventID").count() <= MAX_EXPRESSIONS_PER_SELECT, + "a suppression exceeded the budget: {body}" + ); + } + } + + #[test] + fn an_oversized_exclusion_keeps_the_other_predicates_on_the_selection() { + // The suppressions only name Event IDs. Everything else has to stay on the ") + .nth(1) + .and_then(|rest| rest.split("").next()) + .expect("a selection"); + assert!(selection.contains("Level=1 or Level=2"), "{selection}"); + assert!(!selection.contains("EventID"), "{selection}"); + } + + #[test] + fn a_set_that_fits_the_expression_budget_is_not_split() { + let mut f = filter(); + f.event_ids = (1..=MAX_EXPRESSIONS_PER_SELECT as u32) + .map(|id| EventIdSelector::Single { id }) + .collect(); + assert!(!build_query(&f).expect("builds").contains("")); + } + + #[test] + fn an_apostrophe_is_quoted_with_the_other_delimiter_not_deleted() { + // Deleting it would silently change the value and match nothing, which is worse than an + // error because the filter looks like it worked. + let mut f = filter(); + f.providers = vec!["Bob's Agent".into()]; + assert_eq!( + build_query(&f).expect("builds"), + "*[System[Provider[@Name=\"Bob's Agent\"]]]" + ); + } + + #[test] + fn injected_query_syntax_stays_inside_the_literal() { + let mut f = filter(); + f.providers = vec!["Evil' or '1'='1".into()]; + let query = build_query(&f).expect("builds"); + // The whole value sits inside a double-quoted literal, so none of it is read as syntax. + assert_eq!(query, "*[System[Provider[@Name=\"Evil' or '1'='1\"]]]"); + } + + #[test] + fn a_value_containing_both_delimiters_is_refused_rather_than_mangled() { + let mut f = filter(); + f.providers = vec!["it's \"both\"".into()]; + assert!(matches!( + build_query(&f), + Err(QueryBuildError::UnquotableValue(_)) + )); + } + + #[test] + fn a_bare_xpath_keeps_operators_and_metacharacters_raw() { + // Verified against the service: escaped operators in a bare XPath are rejected with + // "The specified query is invalid". + let mut f = filter(); + f.providers = vec!["A&BD\"E".into()]; + let query = build_query(&f).expect("builds"); + assert_eq!(query, "*[System[Provider[@Name='A&BD\"E']]]"); + assert!(!query.contains("&")); + } + + #[test] + fn embedding_in_a_query_list_escapes_the_whole_expression() { + // The same operators inside a QueryList must be escaped, or the document is not + // well-formed XML and the service rejects it before parsing the XPath. + let mut f = filter(); + f.time = Some(TimeWindow::Last { + milliseconds: 1_000, + }); + f.event_ids = (1..=45).map(|id| EventIdSelector::Single { id }).collect(); + let query = build_query(&f).expect("builds"); + + assert!(query.starts_with("")); + assert!( + query.contains("timediff(@SystemTime) <= 1000"), + "operators must be escaped once embedded: {query}" + ); + assert!( + !query.contains("timediff(@SystemTime) <= 1000"), + "a raw operator inside XML would be malformed: {query}" + ); + } + + #[test] + fn an_ampersand_in_a_provider_is_escaped_only_when_embedded() { + let mut f = filter(); + f.providers = vec!["A&B".into()]; + assert!( + build_query(&f).expect("builds").contains("'A&B'"), + "bare stays raw" + ); + + f.event_ids = (1..=45).map(|id| EventIdSelector::Single { id }).collect(); + assert!( + build_query(&f).expect("builds").contains("'A&B'"), + "embedded gets escaped" + ); + } + + #[test] + fn provider_exclusion_negates_the_clause() { + let mut f = filter(); + f.providers = vec!["Noisy-Provider".into()]; + f.provider_mode = SelectorMode::Exclude; + assert_eq!( + build_query(&f).expect("builds"), + "*[System[Provider[@Name!='Noisy-Provider']]]" + ); + } +} + +#[cfg(test)] +mod wire_tests { + use super::*; + + #[test] + fn a_filter_round_trips_through_the_ipc_boundary() { + let filter = EventQueryFilter { + time: Some(TimeWindow::Last { + milliseconds: 3_600_000, + }), + levels: vec![2, 3], + event_ids: vec![ + EventIdSelector::Single { id: 4624 }, + EventIdSelector::Range { low: 1, high: 9 }, + ], + event_id_mode: SelectorMode::Include, + providers: vec!["ESENT".into()], + provider_mode: SelectorMode::Exclude, + keywords: Some(42), + }; + + let json = serde_json::to_string(&filter).expect("serializes"); + let restored: EventQueryFilter = serde_json::from_str(&json).expect("deserializes"); + + assert_eq!(restored, filter); + assert_eq!( + build_query(&restored).expect("builds"), + build_query(&filter).expect("builds") + ); + } + + #[test] + fn an_absent_field_defaults_rather_than_failing() { + // The frontend sends only what the operator set, so every field must be optional. + let filter: EventQueryFilter = serde_json::from_str("{}").expect("empty object is valid"); + assert!(filter.is_unfiltered()); + assert_eq!(build_query(&filter).expect("builds"), "*"); + } + + #[test] + fn the_wire_shape_is_camel_case_for_typescript() { + let filter = EventQueryFilter { + event_ids: vec![EventIdSelector::Single { id: 1 }], + ..EventQueryFilter::default() + }; + let json = serde_json::to_string(&filter).expect("serializes"); + assert!(json.contains("\"eventIds\""), "{json}"); + assert!(json.contains("\"eventIdMode\""), "{json}"); + assert!(json.contains("\"kind\":\"single\""), "{json}"); + } +} + +#[cfg(test)] +mod service_validated_tests { + //! Golden strings that were executed against a real Windows Event Log service. + //! + //! Every expression below was run on Windows 11 build 26200 against the `Application` channel + //! and accepted. Unit tests can only assert the shape of a string; these pin that shape to + //! forms the service actually parses, so a future change that looks reasonable but is rejected + //! at runtime fails here instead of in front of a user. + //! + //! Three real defects were found this way, none of which any shape-only test could have caught: + //! XML-escaped operators are rejected in a bare XPath, `not(...)` is not in the supported + //! subset at all, and `not Provider[...]` was never valid syntax to begin with. + + use super::*; + + fn assert_query(filter: &EventQueryFilter, expected: &str) { + assert_eq!(build_query(filter).expect("builds"), expected); + } + + #[test] + fn relative_time_matches_the_validated_form() { + assert_query( + &EventQueryFilter { + time: Some(TimeWindow::Last { + milliseconds: 86_400_000, + }), + ..Default::default() + }, + "*[System[TimeCreated[timediff(@SystemTime) <= 86400000]]]", + ); + } + + #[test] + fn absolute_time_matches_the_validated_form() { + assert_query( + &EventQueryFilter { + time: Some(TimeWindow::Between { + from: Some("2026-08-01T00:00:00.000Z".into()), + to: Some("2026-08-10T00:00:00.000Z".into()), + }), + ..Default::default() + }, + "*[System[TimeCreated[@SystemTime >= '2026-08-01T00:00:00.000Z' and @SystemTime <= '2026-08-10T00:00:00.000Z']]]", + ); + } + + #[test] + fn event_id_include_matches_the_validated_form() { + assert_query( + &EventQueryFilter { + event_ids: vec![ + EventIdSelector::Single { id: 1000 }, + EventIdSelector::Range { + low: 300, + high: 330, + }, + ], + ..Default::default() + }, + "*[System[(EventID=1000 or (EventID >= 300 and EventID <= 330))]]", + ); + } + + #[test] + fn event_id_exclude_matches_the_validated_form() { + assert_query( + &EventQueryFilter { + event_ids: vec![EventIdSelector::Single { id: 4688 }], + event_id_mode: SelectorMode::Exclude, + ..Default::default() + }, + "*[System[(EventID!=4688)]]", + ); + } + + #[test] + fn excluding_a_range_uses_its_complement() { + // The subset has no negation to wrap a range in, so the complement is emitted directly. + assert_query( + &EventQueryFilter { + event_ids: vec![EventIdSelector::Range { + low: 300, + high: 330, + }], + event_id_mode: SelectorMode::Exclude, + ..Default::default() + }, + "*[System[((EventID < 300 or EventID > 330))]]", + ); + } + + #[test] + fn excluding_several_ids_requires_all_of_them_to_hold() { + // Joined with "and", not "or": "EventID!=1 or EventID!=2" is true for every event. + assert_query( + &EventQueryFilter { + event_ids: vec![ + EventIdSelector::Single { id: 1 }, + EventIdSelector::Single { id: 2 }, + ], + event_id_mode: SelectorMode::Exclude, + ..Default::default() + }, + "*[System[(EventID!=1 and EventID!=2)]]", + ); + } + + #[test] + fn provider_forms_match_the_validated_forms() { + assert_query( + &EventQueryFilter { + providers: vec!["ESENT".into()], + ..Default::default() + }, + "*[System[Provider[@Name='ESENT']]]", + ); + assert_query( + &EventQueryFilter { + providers: vec!["A".into(), "B".into()], + provider_mode: SelectorMode::Exclude, + ..Default::default() + }, + "*[System[Provider[@Name!='A' and @Name!='B']]]", + ); + } + + #[test] + fn keywords_match_the_validated_form() { + assert_query( + &EventQueryFilter { + keywords: Some(9_223_372_036_854_775_808), + ..Default::default() + }, + "*[System[band(Keywords,9223372036854775808)]]", + ); + } + + #[test] + fn no_emitted_bare_query_contains_an_xml_entity() { + // A bare XPath carrying "<" is rejected by the service. This catches a regression that + // would otherwise only show up as an empty result set on Windows. + let filters = [ + EventQueryFilter { + time: Some(TimeWindow::Last { milliseconds: 1 }), + ..Default::default() + }, + EventQueryFilter { + event_ids: vec![EventIdSelector::Range { low: 1, high: 9 }], + ..Default::default() + }, + EventQueryFilter { + providers: vec!["A&B".into()], + ..Default::default() + }, + ]; + for filter in filters { + let query = build_query(&filter).expect("builds"); + assert!(!query.starts_with("")); + for entity in ["<", ">", "&", """] { + assert!( + !query.contains(entity), + "bare XPath must not contain {entity}: {query}" + ); + } + } + } +} + +#[cfg(test)] +mod expression_budget_tests { + //! The service counts expressions, not selectors. + //! + //! Microsoft documents each XPath as limited to 32 expressions, and a compound expression of + //! more than 20 as requiring a structured XML query. Splitting on a count of selectors gets + //! this wrong whenever selectors are not one expression each. + + use super::*; + + fn singles(count: u32) -> Vec { + (1..=count) + .map(|id| EventIdSelector::Single { id }) + .collect() + } + + #[test] + fn a_range_costs_two_expressions_and_a_degenerate_range_costs_one() { + assert_eq!(EventIdSelector::Single { id: 1 }.expression_cost(), 1); + assert_eq!( + EventIdSelector::Range { low: 1, high: 9 }.expression_cost(), + 2 + ); + assert_eq!( + EventIdSelector::Range { low: 7, high: 7 }.expression_cost(), + 1 + ); + } + + #[test] + fn ranges_are_costed_at_two_expressions_each() { + // Ten selectors either way, but a range is two comparisons. The concrete shape is asserted + // rather than "split or exactly twenty": the disjunction was satisfied by the second half + // and would have passed with the split logic deleted. + let ranges = |count: u32| EventQueryFilter { + event_ids: (0..count) + .map(|i| EventIdSelector::Range { + low: i * 100, + high: i * 100 + 50, + }) + .collect(), + ..Default::default() + }; + + // Ten singles cost ten and stay in one node. + let singles_query = build_query(&EventQueryFilter { + event_ids: singles(10), + ..Default::default() + }) + .expect("builds"); + assert!(!singles_query.contains("")); + + // Ten ranges cost exactly the budget, so they also stay in one node. + let at_budget = build_query(&ranges(10)).expect("builds"); + assert!( + !at_budget.contains(""), + "ten ranges are exactly the budget: {at_budget}" + ); + assert_eq!(at_budget.matches("EventID").count(), 20); + + // Eleven cost 22 and must split, which is what proves ranges are costed as two. + let over_budget = build_query(&ranges(11)).expect("builds"); + assert!( + over_budget.starts_with(""), + "eleven ranges exceed the budget and must split: {over_budget}" + ); + for node in over_budget.split("").next().unwrap_or_default(); + assert!( + body.matches("EventID").count() <= MAX_EXPRESSIONS_PER_SELECT, + "a node exceeded the budget: {body}" + ); + } + } + + #[test] + fn the_other_predicates_consume_budget_too() { + // Six levels plus a time term leave room for far fewer ids in one node. + let f = EventQueryFilter { + levels: vec![0, 1, 2, 3, 4, 5], + time: Some(TimeWindow::Last { milliseconds: 1 }), + event_ids: singles(20), + ..Default::default() + }; + + let query = build_query(&f).expect("builds"); + assert!( + query.contains(""), + "fixed terms must count against the budget: {query}" + ); + } + + #[test] + fn repeated_levels_collapse_instead_of_spending_the_budget_twice() { + // The same level given twice would emit "Level=2 or Level=2": two expressions to say one + // thing, out of a budget of twenty. + let f = EventQueryFilter { + levels: (0..30).map(|n| (n % 6) as u8).collect(), + event_ids: singles(3), + ..Default::default() + }; + + let query = build_query(&f).expect("builds"); + assert_eq!( + query.matches("Level=").count(), + 6, + "six distinct levels, however many times each was given: {query}" + ); + } + + #[test] + fn repeated_providers_collapse_case_insensitively() { + // The service matches provider names without regard to case, so two spellings of one name + // are one term. + let f = EventQueryFilter { + providers: vec![ + "Microsoft-Windows-Kernel-General".into(), + "microsoft-windows-kernel-general".into(), + "Another-Provider".into(), + ], + ..Default::default() + }; + + let query = build_query(&f).expect("builds"); + assert_eq!(query.matches("@Name=").count(), 2, "{query}"); + } + + #[test] + fn a_filter_whose_unsplittable_terms_exceed_one_node_is_refused() { + // Levels, providers, time and keywords repeat in every node, so chunking the Event IDs + // cannot bring them under the limit. Emitting anyway produces a query the service rejects, + // and EvtQueryTolerateQueryErrors turns that refusal into a channel reporting no events: + // the filter looks like it worked and returns nothing. An error the caller can show is the + // only honest outcome. + let f = EventQueryFilter { + providers: (0..30).map(|n| format!("Provider-{n}")).collect(), + event_ids: singles(3), + ..Default::default() + }; + + match build_query(&f) { + Err(QueryBuildError::FilterTooComplex { needed, limit }) => { + // Thirty providers plus the largest single selector, which is one here. + assert_eq!(needed, 31); + assert_eq!(limit, MAX_EXPRESSIONS_PER_SELECT); + } + other => panic!("expected a refusal, got {other:?}"), + } + } + + #[test] + fn a_range_that_cannot_fit_beside_the_fixed_terms_is_refused() { + // A selector cannot be split: a range is two comparisons and stays together. Nineteen + // providers leave a budget of one, and the chunker used to hand that one out anyway, + // emitting a node of twenty-one expressions. The budget exists to hold headroom under the + // measured cliff, so spending it silently is what the refusal now prevents. + let filter = EventQueryFilter { + providers: (0..19).map(|n| format!("Provider-{n}")).collect(), + event_ids: vec![EventIdSelector::Range { + low: 1000, + high: 1050, + }], + ..Default::default() + }; + + match build_query(&filter) { + Err(QueryBuildError::FilterTooComplex { needed, limit }) => { + assert_eq!(needed, 21, "19 fixed plus a range costing 2"); + assert_eq!(limit, MAX_EXPRESSIONS_PER_SELECT); + } + other => panic!("expected a refusal, got {other:?}"), + } + } + + #[test] + fn a_range_that_just_fits_beside_the_fixed_terms_is_built() { + // Eighteen leaves a budget of two, which is exactly a range. The boundary must build. + let filter = EventQueryFilter { + providers: (0..18).map(|n| format!("Provider-{n}")).collect(), + event_ids: vec![EventIdSelector::Range { + low: 1000, + high: 1050, + }], + ..Default::default() + }; + let query = build_query(&filter).expect("builds"); + let expressions = query.matches("EventID").count() + query.matches("@Name").count(); + assert_eq!(expressions, MAX_EXPRESSIONS_PER_SELECT); + } + + #[test] + fn a_filter_exactly_at_the_budget_is_still_built() { + // The boundary is inclusive: twenty is what a node may carry, so it must not be refused. + let f = EventQueryFilter { + providers: (0..MAX_EXPRESSIONS_PER_SELECT) + .map(|n| format!("Provider-{n}")) + .collect(), + ..Default::default() + }; + assert!(build_query(&f).is_ok()); + } +} + +#[cfg(test)] +mod structured_query_service_tests { + //! Structured `` forms executed against a real Windows Event Log service. + //! + //! The bare XPath forms were pinned this way from the start; the structured form was not, and + //! it is the one that only appears once a filter outgrows the expression budget, so it would + //! have reached a user unverified. + //! + //! Run on Windows 11 build 26200 against `Application`, deliberately WITHOUT + //! `EvtQueryTolerateQueryErrors`. With that flag the service accepts a query whose nodes it + //! could not evaluate and quietly returns the rest, so "it worked" proves nothing. Every form + //! below was accepted under strict flags and returned a nonzero count, which rules out both a + //! rejected query and one that silently matches nothing. + //! + //! What this measured, against the documentation: the schema calls `Id` required once a list + //! holds more than one `Query`, but the service does not enforce it. A two-node list without + //! `Id` returned 5240 events, exactly matching the single expression covering the same IDs + //! (5180 + 60). `Id` is emitted regardless, because it costs nothing and a saved custom view + //! is validated against the same schema. + + use super::*; + + fn ids(count: u32) -> Vec { + (1000..1000 + count) + .map(|id| EventIdSelector::Single { id }) + .collect() + } + + #[test] + fn a_split_id_set_matches_the_validated_form() { + let filter = EventQueryFilter { + event_ids: ids(31), + ..Default::default() + }; + assert_eq!( + build_query(&filter).expect("builds"), + "\ + \ + \ + " + ); + } + + #[test] + fn every_node_carries_a_unique_id() { + let filter = EventQueryFilter { + event_ids: ids(100), + ..Default::default() + }; + let query = build_query(&filter).expect("builds"); + let node_count = query.matches("")).count(), + 1, + "id {id} is missing or repeated" + ); + } + } + + #[test] + fn no_node_names_a_channel_path() { + // EvtQuery supplies the channel from its own argument, and the schema requires that if any + // node names a path they all do. Omitting it everywhere is the consistent choice, and was + // accepted by the service. + let filter = EventQueryFilter { + event_ids: ids(31), + ..Default::default() + }; + assert!(!build_query(&filter).expect("builds").contains("Path=")); + } + + #[test] + fn operators_inside_a_structured_query_are_xml_escaped() { + // The inverse of the bare-XPath rule. Raw operators here would not be well-formed XML, and + // escaped operators in a bare XPath are rejected outright. + let filter = EventQueryFilter { + event_ids: ids(31), + levels: vec![1, 2, 3, 4], + time: Some(TimeWindow::Last { + milliseconds: 2_592_000_000, + }), + ..Default::default() + }; + let query = build_query(&filter).expect("builds"); + assert!(query.contains("timediff(@SystemTime) <= 2592000000")); + assert!(!query.contains("<= 2592000000")); + } + + #[test] + fn the_other_predicates_repeat_in_every_node() { + // The service unions the nodes rather than intersecting them, so a predicate that appears + // in only one node would widen the result set instead of narrowing it. + let filter = EventQueryFilter { + event_ids: ids(31), + keywords: Some(0x8020_0000_0000_0000), + ..Default::default() + }; + let query = build_query(&filter).expect("builds"); + assert_eq!( + query.matches("band").count(), + query.matches("").skip(1) { + let body = node.split("").next().unwrap_or_default(); + // Each comparison is one expression; they are joined by `and` or `or`. + let comparisons = body.matches("EventID").count() + + body.matches("Level=").count() + + body.matches("@SystemTime").count() + + body.matches("timediff").count() + + body.matches("band").count(); + assert!( + comparisons <= MAX_EXPRESSIONS_PER_SELECT, + "{comparisons} expressions in one node for {id_count} ids, {levels:?} levels" + ); + } + // Suppressions are nodes too, and are bounded by the same limit. + for node in query.split("").skip(1) { + let body = node.split("").next().unwrap_or_default(); + assert!( + body.matches("EventID").count() <= MAX_EXPRESSIONS_PER_SELECT, + "a suppression exceeded the budget: {body}" + ); + } + } + } + } + } +} diff --git a/crates/cmtraceopen-parser/src/eventmap/apply.rs b/crates/cmtraceopen-parser/src/eventmap/apply.rs new file mode 100644 index 000000000..0b361663a --- /dev/null +++ b/crates/cmtraceopen-parser/src/eventmap/apply.rs @@ -0,0 +1,540 @@ +//! Applying a map to an event. +//! +//! Resolution is deliberately non-fatal. Maps are written against a provider's superset of +//! fields, so an individual event legitimately omits some of them. A missing field is a coverage +//! state, reported on [`MappedValue::unresolved`], never an error and never silently blank. + +use std::collections::BTreeMap; + +use super::model::{EventMap, MapEntry, MapProperty}; +use super::node::EventNode; + +/// One normalized column produced from an event. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MappedValue { + /// The column this fills. + pub property: MapProperty, + /// The rendered text, with every resolved placeholder substituted. + pub text: String, + /// Variables whose paths matched nothing, in template order. + /// + /// Non-empty means `text` still contains their raw `%Name%` placeholders. Callers decide + /// whether to show the column, hide it, or flag it; the engine refuses to guess by blanking + /// the placeholder, which would present a partial value as a complete one. + pub unresolved: Vec, +} + +impl MappedValue { + /// True when every placeholder in the template resolved. + pub fn is_complete(&self) -> bool { + self.unresolved.is_empty() + } +} + +/// The full result of applying a map to one event. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct MappedEvent { + /// Columns in map order. + pub values: Vec, + /// Path expressions that failed to parse, as `(variable, expression)`. + /// + /// A malformed expression is a defect in the map file rather than in the event, so it is + /// surfaced separately instead of being reported as a missing field. + pub invalid_paths: Vec<(String, String)>, +} + +impl MappedEvent { + /// Returns the text written to `property`, if the map produced it. + pub fn value_for(&self, property: &MapProperty) -> Option<&str> { + self.values + .iter() + .find(|value| &value.property == property) + .map(|value| value.text.as_str()) + } +} + +/// Applies every entry of `map` to `event`, which must be the `Event` element. +pub fn apply_map(map: &EventMap, event: &EventNode) -> MappedEvent { + let mut result = MappedEvent::default(); + for entry in &map.maps { + result + .values + .push(apply_entry(map, entry, event, &mut result.invalid_paths)); + } + result +} + +fn apply_entry( + map: &EventMap, + entry: &MapEntry, + event: &EventNode, + invalid_paths: &mut Vec<(String, String)>, +) -> MappedValue { + let template = entry.template(); + let mut resolved: BTreeMap<&str, String> = BTreeMap::new(); + + // Paths and placeholders are parsed once per map, not once per record. This runs for every + // event on a channel, so re-parsing an expression that cannot change was the dominant cost. + for (binding, compiled) in entry.bindings().iter().zip(entry.compiled()) { + // Checked against the original template, never against partially rendered output. + if !template.contains(&compiled.placeholder) { + continue; + } + + let value = match &compiled.path { + Some(path) => path.evaluate(event).map(|value| value.into_owned()), + None => { + let defect = (binding.name.clone(), binding.value.clone()); + if !invalid_paths.contains(&defect) { + invalid_paths.push(defect); + } + None + } + }; + + let translated = value.and_then(|raw| match map.lookup_for(&binding.name) { + Some(lookup) => lookup.translate(&raw), + None => Some(raw), + }); + + if let Some(translated) = translated { + resolved.insert(binding.name.as_str(), translated); + } + } + + // A binding that failed to resolve is simply absent from the map, so the renderer reports it + // alongside placeholders the map never bound at all. Both are the same thing to a reader. + let (text, unresolved) = render(template, &resolved); + + MappedValue { + property: entry.property.clone(), + text, + unresolved, + } +} + +/// Renders `template` in one left-to-right pass, returning the text and any unfilled placeholders. +/// +/// Single-pass matters for correctness, not just speed. Event field content is untrusted: a field +/// whose value is literally `%user%` must not become a substitution target for a later binding, +/// which is exactly what repeated `str::replace` over accumulating output would do. +fn render(template: &str, values: &BTreeMap<&str, String>) -> (String, Vec) { + let mut out = String::with_capacity(template.len()); + let mut unresolved: Vec = Vec::new(); + let mut rest = template; + + while let Some(start) = rest.find('%') { + out.push_str(&rest[..start]); + let after = &rest[start + 1..]; + + let Some(end) = after.find('%') else { + // Unpaired '%': the remainder is literal text. + out.push_str(&rest[start..]); + return (out, unresolved); + }; + + let name = &after[..end]; + if name.is_empty() || name.contains(char::is_whitespace) { + // Not a placeholder. Emit the opening '%' and resume at the character after it, so the + // closing '%' stays available as the opening delimiter of a real placeholder that + // follows, as in "50% off %Cost%". + out.push('%'); + rest = after; + continue; + } + + match values.get(name) { + // The substituted value is appended to `out` and never revisited. + Some(value) => out.push_str(value), + None => { + out.push('%'); + out.push_str(name); + out.push('%'); + if !unresolved.iter().any(|existing| existing == name) { + unresolved.push(name.to_string()); + } + } + } + rest = &after[end + 1..]; + } + + out.push_str(rest); + (out, unresolved) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::eventmap::model::{Lookup, MapEntry, ValueBinding}; + use std::collections::BTreeMap; + + fn event() -> EventNode { + EventNode::new("Event").with_child( + EventNode::new("EventData") + .with_child( + EventNode::new("Data") + .with_attribute("Name", "SubjectUserName") + .with_text("adam"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "SubjectDomainName") + .with_text("TEST"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "BusType") + .with_text("7"), + ), + ) + } + + fn binding(name: &str, field: &str) -> ValueBinding { + ValueBinding { + name: name.to_string(), + value: format!(r#"/Event/EventData/Data[@Name="{field}"]"#), + } + } + + fn map_with(entries: Vec, lookups: Vec) -> EventMap { + EventMap { + author: None, + description: None, + event_id: 4624, + channel: "Security".to_string(), + provider: "Microsoft-Windows-Security-Auditing".to_string(), + maps: entries, + lookups, + } + } + + #[test] + fn substitutes_multiple_placeholders_in_one_template() { + let map = map_with( + vec![MapEntry::new( + MapProperty::UserName, + "%domain%\\%user%".to_string(), + vec![ + binding("domain", "SubjectDomainName"), + binding("user", "SubjectUserName"), + ], + )], + vec![], + ); + + let mapped = apply_map(&map, &event()); + assert_eq!(mapped.value_for(&MapProperty::UserName), Some("TEST\\adam")); + assert!(mapped.values[0].is_complete()); + } + + #[test] + fn missing_field_is_reported_and_leaves_the_placeholder_visible() { + let map = map_with( + vec![MapEntry::new( + MapProperty::PayloadData(1), + "LogonType %LogonType%".to_string(), + vec![binding("LogonType", "LogonType")], + )], + vec![], + ); + + let mapped = apply_map(&map, &event()); + let value = &mapped.values[0]; + assert_eq!(value.unresolved, vec!["LogonType".to_string()]); + assert!(!value.is_complete()); + assert_eq!(value.text, "LogonType %LogonType%"); + } + + #[test] + fn lookup_translates_a_bound_variable() { + let map = map_with( + vec![MapEntry::new( + MapProperty::PayloadData(1), + "Bus: %BusType%".to_string(), + vec![binding("BusType", "BusType")], + )], + vec![Lookup { + name: "BusType".to_string(), + default: Some("Unknown code".to_string()), + values: BTreeMap::from([("7".to_string(), "USB".to_string())]), + }], + ); + + let mapped = apply_map(&map, &event()); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("Bus: USB") + ); + } + + #[test] + fn lookup_default_applies_to_an_unknown_code() { + let map = map_with( + vec![MapEntry::new( + MapProperty::PayloadData(1), + "Bus: %BusType%".to_string(), + vec![binding("BusType", "BusType")], + )], + vec![Lookup { + name: "BusType".to_string(), + default: Some("Unknown code".to_string()), + values: BTreeMap::new(), + }], + ); + + let mapped = apply_map(&map, &event()); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("Bus: Unknown code") + ); + } + + #[test] + fn a_template_with_no_placeholders_is_emitted_verbatim() { + let map = map_with( + vec![MapEntry::new( + MapProperty::PayloadData(1), + "Screen saver invoked".to_string(), + vec![], + )], + vec![], + ); + + let mapped = apply_map(&map, &event()); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("Screen saver invoked") + ); + assert!(mapped.values[0].is_complete()); + } + + #[test] + fn a_placeholder_with_no_binding_is_reported_unresolved() { + let map = map_with( + vec![MapEntry::new( + MapProperty::PayloadData(1), + "Value %NeverBound%".to_string(), + vec![], + )], + vec![], + ); + + let mapped = apply_map(&map, &event()); + assert_eq!(mapped.values[0].unresolved, vec!["NeverBound".to_string()]); + } + + #[test] + fn a_malformed_path_is_a_map_defect_not_a_missing_field() { + let map = map_with( + vec![MapEntry::new( + MapProperty::PayloadData(1), + "%broken%".to_string(), + vec![ValueBinding { + name: "broken".to_string(), + value: "EventData/Data".to_string(), + }], + )], + vec![], + ); + + let mapped = apply_map(&map, &event()); + assert_eq!( + mapped.invalid_paths, + vec![("broken".to_string(), "EventData/Data".to_string())] + ); + assert_eq!(mapped.values[0].unresolved, vec!["broken".to_string()]); + } + + #[test] + fn a_field_value_containing_a_placeholder_is_not_re_substituted() { + // Event content is untrusted. If rendering re-scanned its own output, a field whose text + // happens to be "%user%" would be replaced by a later binding. + let event = EventNode::new("Event").with_child( + EventNode::new("EventData") + .with_child( + EventNode::new("Data") + .with_attribute("Name", "SubjectDomainName") + .with_text("%user%"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "SubjectUserName") + .with_text("adam"), + ), + ); + let map = map_with( + vec![MapEntry::new( + MapProperty::UserName, + "%domain%\\%user%".to_string(), + vec![ + binding("domain", "SubjectDomainName"), + binding("user", "SubjectUserName"), + ], + )], + vec![], + ); + + let mapped = apply_map(&map, &event); + assert_eq!( + mapped.value_for(&MapProperty::UserName), + Some("%user%\\adam") + ); + } + + #[test] + fn a_literal_percent_does_not_hide_the_placeholder_that_follows_it() { + let map = map_with( + vec![MapEntry::new( + MapProperty::PayloadData(1), + "50% off %Cost%".to_string(), + vec![], + )], + vec![], + ); + + let mapped = apply_map(&map, &event()); + let value = &mapped.values[0]; + assert_eq!(value.unresolved, vec!["Cost".to_string()]); + assert!(!value.is_complete()); + assert_eq!(value.text, "50% off %Cost%"); + } + + #[test] + fn a_literal_percent_survives_when_a_later_placeholder_resolves() { + let map = map_with( + vec![MapEntry::new( + MapProperty::PayloadData(1), + "50% off for %user%".to_string(), + vec![binding("user", "SubjectUserName")], + )], + vec![], + ); + + let mapped = apply_map(&map, &event()); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("50% off for adam") + ); + assert!(mapped.values[0].is_complete()); + } + + #[test] + fn an_unpaired_trailing_percent_is_preserved_verbatim() { + let map = map_with( + vec![MapEntry::new( + MapProperty::PayloadData(1), + "complete: 100%".to_string(), + vec![], + )], + vec![], + ); + + let mapped = apply_map(&map, &event()); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("complete: 100%") + ); + assert!(mapped.values[0].is_complete()); + } + + #[test] + fn an_unused_binding_does_not_affect_the_result() { + let map = map_with( + vec![MapEntry::new( + MapProperty::PayloadData(1), + "User %user%".to_string(), + vec![ + binding("user", "SubjectUserName"), + binding("unused", "DoesNotExist"), + ], + )], + vec![], + ); + + let mapped = apply_map(&map, &event()); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("User adam") + ); + assert!(mapped.values[0].is_complete()); + } + + #[test] + fn applying_an_entry_twice_gives_the_same_answer() { + // The compiled bindings are memoized and a OnceLock never invalidates, so the risk is a + // cache that outlives the content it was built from. The inputs are private, which makes + // that unrepresentable rather than unlikely: there is no way to reach this entry's + // template or bindings and change them between these two calls. + let map = map_with( + vec![MapEntry::new( + MapProperty::PayloadData(1), + "%first% then %second%".to_string(), + vec![binding("first", "A"), binding("second", "B")], + )], + vec![], + ); + let event = EventNode::new("Event").with_child( + EventNode::new("EventData") + .with_child( + EventNode::new("Data") + .with_attribute("Name", "A") + .with_text("alpha"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "B") + .with_text("beta"), + ), + ); + + let cold = apply_map(&map, &event); + let warm = apply_map(&map, &event); + assert_eq!(cold, warm); + assert_eq!( + warm.value_for(&MapProperty::PayloadData(1)), + Some("alpha then beta") + ); + } + + #[test] + fn a_rebuilt_entry_compiles_its_own_content() { + // Rebuilding is the only way to get different content now, so the guarantee that matters + // is that a new entry never inherits an older one's compiled paths. + let event = EventNode::new("Event").with_child( + EventNode::new("EventData").with_child( + EventNode::new("Data") + .with_attribute("Name", "A") + .with_text("alpha"), + ), + ); + + let first = map_with( + vec![MapEntry::new( + MapProperty::PayloadData(1), + "%only%".to_string(), + vec![binding("only", "A")], + )], + vec![], + ); + assert_eq!( + apply_map(&first, &event).value_for(&MapProperty::PayloadData(1)), + Some("alpha") + ); + + // Same placeholder, different path: the second entry must resolve by its own expression. + let second = map_with( + vec![MapEntry::new( + MapProperty::PayloadData(1), + "%only%".to_string(), + vec![binding("only", "missing")], + )], + vec![], + ); + let mapped = apply_map(&second, &event); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("%only%"), + "the rebuilt entry must not resolve through the first entry's path" + ); + } +} diff --git a/crates/cmtraceopen-parser/src/eventmap/mod.rs b/crates/cmtraceopen-parser/src/eventmap/mod.rs new file mode 100644 index 000000000..e9f7b9f9e --- /dev/null +++ b/crates/cmtraceopen-parser/src/eventmap/mod.rs @@ -0,0 +1,210 @@ +//! Normalizing Windows event data into stable columns. +//! +//! Every event ID carries a different `EventData` shape, which is why event logs resist tabular +//! display. EvtxECmd solved this with community "maps" that project each event's fields into a +//! fixed set of columns. This module implements that schema so the existing corpus of upstream +//! maps works unmodified, and so maps written here work in EvtxECmd and Timeline Explorer. +//! +//! Two boundaries keep this crate pure and wasm32-compatible: +//! +//! - **No XML dependency.** Callers convert whatever they hold into [`EventNode`]. +//! - **No YAML dependency.** The schema derives `serde::Deserialize`, so the host layer picks the +//! format. Upstream maps are YAML; the fixtures here are the same maps as JSON. +//! +//! ``` +//! use cmtraceopen_parser::eventmap::{apply_map, EventMap, EventNode, MapProperty}; +//! +//! let map: EventMap = serde_json::from_str(r#"{ +//! "EventId": 9701, +//! "Channel": "Microsoft-Windows-Shell-Core/Operational", +//! "Provider": "Microsoft-Windows-Shell-Core", +//! "Maps": [{ +//! "Property": "PayloadData1", +//! "PropertyValue": "%PayloadData1%", +//! "Values": [{ "Name": "PayloadData1", "Value": "/Event/EventData/Data" }] +//! }] +//! }"#).unwrap(); +//! +//! let event = EventNode::new("Event").with_child( +//! EventNode::new("EventData").with_child(EventNode::new("Data").with_text("RunOnceEx")), +//! ); +//! +//! let mapped = apply_map(&map, &event); +//! assert_eq!(mapped.value_for(&MapProperty::PayloadData(1)), Some("RunOnceEx")); +//! ``` + +mod apply; +mod model; +mod node; +mod path; + +pub use apply::{apply_map, MappedEvent, MappedValue}; +pub use model::{EventMap, Lookup, MapEntry, MapProperty, ValueBinding}; +pub use node::EventNode; +pub use path::{PathError, Predicate, Step, ValuePath}; + +use std::collections::HashMap; + +/// Identity of a map: channel, provider, and event ID. +/// +/// The file name is not part of identity upstream, and neither is it here. Channel and provider +/// are compared ASCII case-insensitively, because a case mismatch would silently drop the mapping +/// for every event of that type rather than fail loudly. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct MapKey { + channel: String, + provider: String, + event_id: u32, +} + +impl MapKey { + fn new(channel: &str, provider: &str, event_id: u32) -> Self { + Self { + channel: channel.trim().to_ascii_lowercase(), + provider: provider.trim().to_ascii_lowercase(), + event_id, + } + } +} + +/// A set of maps, resolved by channel, provider, and event ID. +#[derive(Debug, Clone, Default)] +pub struct MapRegistry { + maps: HashMap, +} + +impl MapRegistry { + /// Creates an empty registry. + pub fn new() -> Self { + Self::default() + } + + /// Inserts `map`, returning any map it replaced. + /// + /// Upstream loads maps in alphabetical file order so that a `1_`-prefixed copy overrides the + /// original. Load order is the caller's concern; last insert wins here. + pub fn insert(&mut self, map: EventMap) -> Option { + let key = MapKey::new(&map.channel, &map.provider, map.event_id); + self.maps.insert(key, map) + } + + /// Finds the map for an event, if one exists. + pub fn find(&self, channel: &str, provider: &str, event_id: u32) -> Option<&EventMap> { + self.maps.get(&MapKey::new(channel, provider, event_id)) + } + + /// Number of maps held. + pub fn len(&self) -> usize { + self.maps.len() + } + + /// True when no maps are held. + pub fn is_empty(&self) -> bool { + self.maps.is_empty() + } + + /// Applies the matching map to `event`, if one is registered. + pub fn apply( + &self, + channel: &str, + provider: &str, + event_id: u32, + event: &EventNode, + ) -> Option { + self.find(channel, provider, event_id) + .map(|map| apply_map(map, event)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn map(channel: &str, provider: &str, event_id: u32) -> EventMap { + EventMap { + author: None, + description: None, + event_id, + channel: channel.to_string(), + provider: provider.to_string(), + maps: Vec::new(), + lookups: Vec::new(), + } + } + + #[test] + fn resolves_by_channel_provider_and_event_id() { + let mut registry = MapRegistry::new(); + registry.insert(map("Security", "Microsoft-Windows-Security-Auditing", 4624)); + + assert!(registry + .find("Security", "Microsoft-Windows-Security-Auditing", 4624) + .is_some()); + assert!(registry + .find("Security", "Microsoft-Windows-Security-Auditing", 4625) + .is_none()); + assert!(registry + .find("System", "Microsoft-Windows-Security-Auditing", 4624) + .is_none()); + } + + #[test] + fn channel_and_provider_matching_is_case_insensitive() { + let mut registry = MapRegistry::new(); + registry.insert(map( + "Microsoft-Windows-Shell-Core/Operational", + "Microsoft-Windows-Shell-Core", + 9701, + )); + + assert!(registry + .find( + "microsoft-windows-shell-core/operational", + "MICROSOFT-WINDOWS-SHELL-CORE", + 9701 + ) + .is_some()); + } + + #[test] + fn the_same_event_id_on_different_channels_stays_distinct() { + let mut registry = MapRegistry::new(); + registry.insert(map("Security", "Provider-A", 4624)); + registry.insert(map("System", "Provider-A", 4624)); + + assert_eq!(registry.len(), 2); + } + + #[test] + fn reinserting_the_same_identity_replaces_and_returns_the_previous_map() { + let mut registry = MapRegistry::new(); + let mut first = map("Security", "Provider-A", 4624); + first.description = Some("original".to_string()); + registry.insert(first); + + let mut override_map = map("Security", "Provider-A", 4624); + override_map.description = Some("override".to_string()); + let replaced = registry.insert(override_map); + + assert_eq!( + replaced.and_then(|m| m.description).as_deref(), + Some("original") + ); + assert_eq!(registry.len(), 1); + assert_eq!( + registry + .find("Security", "Provider-A", 4624) + .and_then(|m| m.description.as_deref()), + Some("override") + ); + } + + #[test] + fn an_empty_registry_maps_nothing() { + let registry = MapRegistry::new(); + assert!(registry.is_empty()); + assert!(registry + .apply("Security", "Provider-A", 4624, &EventNode::new("Event")) + .is_none()); + } +} diff --git a/crates/cmtraceopen-parser/src/eventmap/model.rs b/crates/cmtraceopen-parser/src/eventmap/model.rs new file mode 100644 index 000000000..9d5deb3c3 --- /dev/null +++ b/crates/cmtraceopen-parser/src/eventmap/model.rs @@ -0,0 +1,315 @@ +//! The EvtxECmd map schema. +//! +//! Deliberately derives `serde::Deserialize` rather than depending on a YAML crate. Upstream maps +//! are YAML, but this crate is pure and wasm32-compatible, so format-specific loading belongs in +//! the host layer. Any serde format that produces these field names works, which keeps YAML out +//! of the parser crate and lets tests drive the real corpus through `serde_json`. + +use std::collections::BTreeMap; +use std::sync::OnceLock; + +use serde::{Deserialize, Deserializer}; + +use super::path::ValuePath; + +/// A normalized output column that a map entry writes into. +/// +/// The upstream corpus is not perfectly consistent: four maps spell the target `Username` rather +/// than `UserName`, so parsing is ASCII case-insensitive. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +// Growable: this build does not know every variant a newer schema will define. +// Marking it now keeps adding one a minor change; after the first release that +// exposes the type, adding the attribute is itself breaking. +#[non_exhaustive] +pub enum MapProperty { + /// The acting or target account. + UserName, + /// The remote host, typically a workstation name and address. + RemoteHost, + /// Process, command line, service, or scheduled-task detail. + ExecutableInfo, + /// One of the six generic overflow slots, numbered 1 through 6. + PayloadData(u8), + /// A target this build does not know, preserved rather than dropped so a community map + /// contributed against a newer schema is not silently discarded. + Other(String), +} + +impl MapProperty { + /// Parses a `Property` value, ASCII case-insensitively. + pub fn parse(raw: &str) -> Self { + let trimmed = raw.trim(); + if trimmed.eq_ignore_ascii_case("username") { + return Self::UserName; + } + if trimmed.eq_ignore_ascii_case("remotehost") { + return Self::RemoteHost; + } + if trimmed.eq_ignore_ascii_case("executableinfo") { + return Self::ExecutableInfo; + } + if let Some(suffix) = trimmed + .get(..11) + .filter(|prefix| prefix.eq_ignore_ascii_case("payloaddata")) + .and_then(|_| trimmed.get(11..)) + { + if let Ok(slot @ 1..=6) = suffix.parse::() { + return Self::PayloadData(slot); + } + } + Self::Other(trimmed.to_string()) + } +} + +impl<'de> Deserialize<'de> for MapProperty { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + Ok(Self::parse(&raw)) + } +} + +/// Binds a template variable to a location in the event. +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "PascalCase")] +pub struct ValueBinding { + /// Variable name, referenced as `%Name%` in the owning entry's `PropertyValue`. + pub name: String, + /// The path expression locating the value. + pub value: String, +} + +/// A translation table turning raw codes into readable text. +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "PascalCase")] +pub struct Lookup { + /// Matches the [`ValueBinding::name`] this table applies to. + pub name: String, + /// Text used when the raw value is absent from `values`. + #[serde(default)] + pub default: Option, + /// Raw value to readable text. + #[serde(default)] + pub values: BTreeMap, +} + +impl Lookup { + /// Translates `raw`, falling back to [`Lookup::default`] when it is not in the table. + pub fn translate(&self, raw: &str) -> Option { + self.values + .get(raw) + .cloned() + .or_else(|| self.default.clone()) + } +} + +/// A binding with its path expression already parsed and its placeholder already formatted. +/// +/// Both are constant for the life of the map, and applying a map happens once per record. Parsing +/// the same expression a million times to get a million identical results is the cost this avoids. +#[derive(Debug, Clone)] +pub struct CompiledBinding { + /// `%Name%`, formatted once. + pub placeholder: String, + /// The parsed path, or `None` when the expression is malformed. + /// + /// A malformed expression is kept as a value rather than dropped so the applier can still + /// report it: a map with a typo in it is a defect an operator needs told about, not a binding + /// that silently resolves to nothing. + pub path: Option, +} + +/// One output column produced from an event. +/// +/// `PartialEq` compares the deserialized content only. The compiled cache below is derived from +/// it, so two entries that describe the same mapping are equal whether or not either has been +/// applied yet. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct MapEntry { + /// Which normalized column this writes. + pub property: MapProperty, + /// Template containing `%Name%` placeholders. + /// + /// Private, with [`template`](Self::template) to read it. The compiled cache below is derived + /// from this field and from `values`, and a `OnceLock` does not invalidate: a caller who + /// mutated either after the first application would leave a complete but stale cache, and the + /// applier would pair current bindings with compiled records describing the old ones. Rather + /// than add an invalidation path nobody would remember to call, the inputs cannot be changed. + property_value: String, + /// Variables available to the template. + /// + /// Private for the same reason as `property_value`. + #[serde(default)] + values: Vec, + /// Built on first use and reused for every later record. + /// + /// Memoized here rather than compiled by whoever owns the map, because a separate compile step + /// is one a caller can forget: a map that skipped it would resolve nothing and look like a map + /// that simply did not match. + #[serde(skip)] + compiled: OnceLock>, +} + +impl MapEntry { + /// Builds an entry. + /// + /// The compiled cache is not a constructor argument: it is derived from these fields and is + /// built on first use, so there is no state a caller could supply inconsistently. + pub fn new(property: MapProperty, property_value: String, values: Vec) -> Self { + Self { + property, + property_value, + values, + compiled: OnceLock::new(), + } + } + + /// The template this entry renders, with its `%Name%` placeholders. + pub fn template(&self) -> &str { + &self.property_value + } + + /// The variables the template can reference, in declaration order. + /// + /// Positionally aligned with [`compiled`](Self::compiled); both derive from the same field and + /// neither can change after construction. + pub fn bindings(&self) -> &[ValueBinding] { + &self.values + } + + /// The entry's bindings, parsed once. + /// + /// Every binding is compiled, including any whose placeholder the template never references. + /// Compiling all of them keeps this positionally aligned with [`bindings`](Self::bindings), + /// which is what lets the applier zip the two; skipping some would misalign the pair. The + /// applier decides separately which to *evaluate*, and an unreferenced binding is never + /// evaluated or reported. + pub fn compiled(&self) -> &[CompiledBinding] { + self.compiled.get_or_init(|| { + self.values + .iter() + .map(|binding| CompiledBinding { + placeholder: format!("%{}%", binding.name), + path: ValuePath::parse(&binding.value).ok(), + }) + .collect() + }) + } +} + +impl PartialEq for MapEntry { + fn eq(&self, other: &Self) -> bool { + self.property == other.property + && self.property_value == other.property_value + && self.values == other.values + } +} + +impl Eq for MapEntry {} + +/// A parsed EvtxECmd map file. +/// +/// Identity is `(channel, provider, event_id)`, not the file name, matching upstream. +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "PascalCase")] +pub struct EventMap { + /// Map author, carried through for attribution. + #[serde(default)] + pub author: Option, + /// What the mapped event means. + #[serde(default)] + pub description: Option, + /// The event ID this map applies to. + pub event_id: u32, + /// The channel this map applies to, for example `Security`. + pub channel: String, + /// The provider this map applies to. + pub provider: String, + /// Output columns, evaluated in order. + #[serde(default)] + pub maps: Vec, + /// Translation tables referenced by variable name. + #[serde(default)] + pub lookups: Vec, +} + +impl EventMap { + /// Returns the lookup table bound to `variable`, if any. + pub fn lookup_for(&self, variable: &str) -> Option<&Lookup> { + self.lookups + .iter() + .find(|lookup| lookup.name.eq_ignore_ascii_case(variable)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_known_property_targets() { + assert_eq!(MapProperty::parse("UserName"), MapProperty::UserName); + assert_eq!(MapProperty::parse("RemoteHost"), MapProperty::RemoteHost); + assert_eq!( + MapProperty::parse("ExecutableInfo"), + MapProperty::ExecutableInfo + ); + assert_eq!( + MapProperty::parse("PayloadData4"), + MapProperty::PayloadData(4) + ); + } + + #[test] + fn property_parsing_tolerates_the_corpus_case_inconsistency() { + // Four upstream maps spell this "Username"; treating that as unknown would silently drop + // the account column for those events. + assert_eq!(MapProperty::parse("Username"), MapProperty::UserName); + assert_eq!( + MapProperty::parse("payloaddata2"), + MapProperty::PayloadData(2) + ); + } + + #[test] + fn unknown_and_out_of_range_targets_are_preserved() { + assert_eq!( + MapProperty::parse("PayloadData7"), + MapProperty::Other("PayloadData7".to_string()) + ); + assert_eq!( + MapProperty::parse("SomethingNew"), + MapProperty::Other("SomethingNew".to_string()) + ); + } + + #[test] + fn property_parsing_does_not_panic_on_short_input() { + assert_eq!(MapProperty::parse(""), MapProperty::Other(String::new())); + assert_eq!( + MapProperty::parse("Pay"), + MapProperty::Other("Pay".to_string()) + ); + } + + #[test] + fn lookup_falls_back_to_default() { + let lookup = Lookup { + name: "BusType".to_string(), + default: Some("Unknown code".to_string()), + values: BTreeMap::from([("7".to_string(), "USB".to_string())]), + }; + assert_eq!(lookup.translate("7").as_deref(), Some("USB")); + assert_eq!(lookup.translate("99").as_deref(), Some("Unknown code")); + } + + #[test] + fn lookup_without_default_returns_none_for_unknown_codes() { + let lookup = Lookup { + name: "BusType".to_string(), + default: None, + values: BTreeMap::new(), + }; + assert_eq!(lookup.translate("7"), None); + } +} diff --git a/crates/cmtraceopen-parser/src/eventmap/node.rs b/crates/cmtraceopen-parser/src/eventmap/node.rs new file mode 100644 index 000000000..0bc5d6545 --- /dev/null +++ b/crates/cmtraceopen-parser/src/eventmap/node.rs @@ -0,0 +1,118 @@ +//! A minimal, XML-free event tree. +//! +//! The map engine has to read values out of a rendered Windows event, but this crate is pure +//! Rust and wasm32-compatible, so it must not depend on an XML reader. Callers convert whatever +//! they already have (rendered event XML in `src-tauri`, an `evtx` record, a test literal) into +//! [`EventNode`] and hand that across. + +/// A single element in a rendered event. +/// +/// `name` is the local element name with any namespace prefix already stripped, because event +/// XML paths in maps are written without prefixes (`/Event/EventData/Data`). +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct EventNode { + /// Local element name, for example `Data`. + pub name: String, + /// Attributes in document order, for example `("Name", "LogonType")`. + pub attributes: Vec<(String, String)>, + /// Element text content, if the element has any. + pub text: Option, + /// Child elements in document order. + pub children: Vec, +} + +impl EventNode { + /// Creates an element with no attributes, text, or children. + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + ..Self::default() + } + } + + /// Sets the element's text content. + #[must_use] + pub fn with_text(mut self, text: impl Into) -> Self { + self.text = Some(text.into()); + self + } + + /// Appends an attribute. + #[must_use] + pub fn with_attribute(mut self, name: impl Into, value: impl Into) -> Self { + self.attributes.push((name.into(), value.into())); + self + } + + /// Appends a child element. + #[must_use] + pub fn with_child(mut self, child: EventNode) -> Self { + self.children.push(child); + self + } + + /// Returns the value of `name`, compared case-sensitively as event XML declares it. + pub fn attribute(&self, name: &str) -> Option<&str> { + self.attributes + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.as_str()) + } + + /// Returns the child elements called `name`, in document order. + /// + /// `name` carries its own lifetime so callers can pass a short-lived borrow, such as a field + /// of a path step being walked, without tying it to how long this node lives. + pub fn children_named<'a, 'n>( + &'a self, + name: &'n str, + ) -> impl Iterator + use<'a, 'n> { + self.children.iter().filter(move |child| child.name == name) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample() -> EventNode { + EventNode::new("EventData") + .with_child( + EventNode::new("Data") + .with_attribute("Name", "SubjectUserName") + .with_text("adam"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "LogonType") + .with_text("10"), + ) + } + + #[test] + fn attribute_lookup_is_case_sensitive() { + let node = sample(); + let first = node.children.first().expect("first child"); + assert_eq!(first.attribute("Name"), Some("SubjectUserName")); + assert_eq!(first.attribute("name"), None); + } + + #[test] + fn children_named_preserves_document_order() { + let node = sample(); + let texts: Vec<_> = node + .children_named("Data") + .map(|child| child.text.as_deref().unwrap_or_default()) + .collect(); + assert_eq!(texts, vec!["adam", "10"]); + } + + #[test] + fn children_named_ignores_other_elements() { + let node = EventNode::new("Event") + .with_child(EventNode::new("System")) + .with_child(EventNode::new("EventData")); + assert_eq!(node.children_named("Data").count(), 0); + assert_eq!(node.children_named("System").count(), 1); + } +} diff --git a/crates/cmtraceopen-parser/src/eventmap/path.rs b/crates/cmtraceopen-parser/src/eventmap/path.rs new file mode 100644 index 000000000..fe10afeb0 --- /dev/null +++ b/crates/cmtraceopen-parser/src/eventmap/path.rs @@ -0,0 +1,492 @@ +//! The tiny path language used by EvtxECmd map `Value` expressions. +//! +//! Map files describe where a value lives with what looks like XPath, but the corpus only uses a +//! very small subset. Measured across all 468 upstream maps (1,837 expressions): +//! +//! | Shape | Count | +//! |---|---| +//! | `/Event/EventData/Data[@Name="X"]` | 1,441 | +//! | `/Event/UserData//` | 204 | +//! | `/Event/EventData/Data` | 176 | +//! | `/Event/System/...`, including `Correlation/@ActivityID` | 12 | +//! | `/Event/EventData/Data[N]` | 3 | +//! | `/Event/EventData` | 1 | +//! +//! So this is an absolute element path where each step may carry an attribute-equality or +//! 1-based index predicate, optionally ending in an attribute selector. A general XPath engine +//! would be far more machinery than the grammar justifies. + +use std::borrow::Cow; + +use thiserror::Error; + +use super::node::EventNode; + +/// A failure to parse a map `Value` expression. +#[derive(Debug, Error, PartialEq, Eq)] +// Growable: this build does not know every variant a newer schema will define. +// Marking it now keeps adding one a minor change; after the first release that +// exposes the type, adding the attribute is itself breaking. +#[non_exhaustive] +pub enum PathError { + /// The expression did not start with `/`. + #[error("value path must be absolute, starting with '/': {0}")] + NotAbsolute(String), + /// The expression had no element steps. + #[error("value path has no steps: {0}")] + Empty(String), + /// A predicate was opened but not closed, or was not understood. + #[error("value path has a malformed predicate in step '{step}': {path}")] + MalformedPredicate { + /// The whole expression, so the map entry it came from can be identified. + path: String, + /// The single step that would not parse, which is the part to correct. + step: String, + }, + /// A step was empty, from a doubled or dangling separator, or an attribute with no name. + /// + /// Refused rather than skipped: `/Event//EventData/Data` is a typo, and quietly reading it as + /// `/Event/EventData/Data` would apply a map the author did not write. + #[error("value path has an empty step '{step}': {path}")] + MalformedStep { + /// The whole expression, so the map entry it came from can be identified. + path: String, + /// The offending step, empty for a doubled separator. + step: String, + }, + /// An attribute selector appeared somewhere other than the final step. + #[error("value path may only select an attribute in its final step: {0}")] + MisplacedAttribute(String), +} + +/// Narrows which sibling a step selects. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Predicate { + /// `[@Name="value"]`, selecting the first sibling whose attribute matches. + AttributeEquals { + /// Attribute to compare, without the leading `@`. + name: String, + /// Value it must equal, already unquoted. + value: String, + }, + /// `[n]`, a 1-based index across same-named siblings, as XPath numbers them. + Index(usize), +} + +/// One element step of a path. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Step { + /// Element name to match. + pub name: String, + /// Optional narrowing predicate. + pub predicate: Option, +} + +/// A parsed map `Value` expression. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ValuePath { + steps: Vec, + attribute: Option, +} + +impl ValuePath { + /// Parses an expression such as `/Event/EventData/Data[@Name="LogonType"]`. + pub fn parse(expression: &str) -> Result { + let trimmed = expression.trim(); + let Some(body) = trimmed.strip_prefix('/') else { + return Err(PathError::NotAbsolute(expression.to_string())); + }; + + if body.trim().is_empty() { + return Err(PathError::Empty(expression.to_string())); + } + + let mut steps = Vec::new(); + let mut attribute = None; + // Not filtered. Dropping empty segments quietly accepted `/Event//EventData/Data` and + // `/Event/`, which are malformed paths that would then evaluate as though they were the + // path the author meant. A map with a typo should be reported, not silently reinterpreted. + let segments: Vec<&str> = body.split('/').collect(); + + for (index, segment) in segments.iter().enumerate() { + if segment.is_empty() { + // A single trailing slash is the one benign case: `/Event/EventData/` names the + // same element as without it. + if index + 1 == segments.len() && !steps.is_empty() { + continue; + } + return Err(PathError::MalformedStep { + path: expression.to_string(), + step: String::new(), + }); + } + if let Some(attribute_name) = segment.strip_prefix('@') { + if index + 1 != segments.len() { + return Err(PathError::MisplacedAttribute(expression.to_string())); + } + if attribute_name.is_empty() { + return Err(PathError::MalformedStep { + path: expression.to_string(), + step: segment.to_string(), + }); + } + attribute = Some(attribute_name.to_string()); + continue; + } + steps.push(parse_step(segment, expression)?); + } + + if steps.is_empty() { + return Err(PathError::Empty(expression.to_string())); + } + + Ok(Self { steps, attribute }) + } + + /// Resolves the path against `root`, which must be the `Event` element itself. + /// + /// Returns `None` when any step fails to match, which is the normal case for an optional + /// field rather than an error: maps are written against a provider's superset of fields and + /// individual events legitimately omit some of them. + pub fn evaluate<'a>(&self, root: &'a EventNode) -> Option> { + let first = self.steps.first()?; + if first.name != root.name || first.predicate.is_some() { + return None; + } + + // Every step but the last narrows to a single container. Only the final step can select + // a repeated set, because joining containers has no meaning. + let mut current = root; + let mut remaining = &self.steps[1..]; + while remaining.len() > 1 { + current = select_one(current, &remaining[0])?; + remaining = &remaining[1..]; + } + + match remaining.first() { + None => read(current, self.attribute.as_deref()), + Some(step) => read_final(current, step, self.attribute.as_deref()), + } + } +} + +/// Separator EvtxECmd uses when a bare step matches repeated elements. +/// +/// Verified against EvtxECmd itself rather than assumed. A probe map binding +/// `/Event/EventData/Data` was run over a real `ESENT` event ID 326 record carrying nine unnamed +/// `` children: the emitted `PayloadData1` was 1,712 characters longer than the first +/// element alone, and the bytes between elements were 44, 32. +const REPEATED_ELEMENT_SEPARATOR: &str = ", "; + +fn read<'a>(node: &'a EventNode, attribute: Option<&str>) -> Option> { + match attribute { + Some(name) => node.attribute(name).map(Cow::Borrowed), + None => node.text.as_deref().map(Cow::Borrowed), + } +} + +fn read_final<'a>( + parent: &'a EventNode, + step: &Step, + attribute: Option<&str>, +) -> Option> { + if step.predicate.is_some() { + return read(select_one(parent, step)?, attribute); + } + + let matches: Vec<&EventNode> = parent + .children + .iter() + .filter(|child| child.name == step.name) + .collect(); + + match matches.as_slice() { + [] => None, + [only] => read(only, attribute), + // An attribute selector still reads one element; only text content is joined. + many => match attribute { + Some(name) => many[0].attribute(name).map(Cow::Borrowed), + None => Some(Cow::Owned( + many.iter() + .map(|node| node.text.as_deref().unwrap_or_default()) + .collect::>() + .join(REPEATED_ELEMENT_SEPARATOR), + )), + }, + } +} + +fn parse_step(segment: &str, expression: &str) -> Result { + let Some(open) = segment.find('[') else { + return Ok(Step { + name: segment.to_string(), + predicate: None, + }); + }; + + let malformed = || PathError::MalformedPredicate { + path: expression.to_string(), + step: segment.to_string(), + }; + + if !segment.ends_with(']') { + return Err(malformed()); + } + + let name = segment[..open].to_string(); + let inner = &segment[open + 1..segment.len() - 1]; + + let predicate = if let Some(rest) = inner.strip_prefix('@') { + let (attribute, raw_value) = rest.split_once('=').ok_or_else(malformed)?; + let value = raw_value + .strip_prefix('"') + .and_then(|v| v.strip_suffix('"')) + .or_else(|| { + raw_value + .strip_prefix('\'') + .and_then(|v| v.strip_suffix('\'')) + }) + .ok_or_else(malformed)?; + Predicate::AttributeEquals { + name: attribute.to_string(), + value: value.to_string(), + } + } else { + let index: usize = inner.parse().map_err(|_| malformed())?; + if index == 0 { + return Err(malformed()); + } + Predicate::Index(index) + }; + + Ok(Step { + name, + predicate: Some(predicate), + }) +} + +fn select_one<'a>(parent: &'a EventNode, step: &Step) -> Option<&'a EventNode> { + let mut candidates = parent.children_named(&step.name); + match &step.predicate { + None => candidates.next(), + Some(Predicate::Index(index)) => candidates.nth(index - 1), + Some(Predicate::AttributeEquals { name, value }) => { + candidates.find(|child| child.attribute(name) == Some(value.as_str())) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn event() -> EventNode { + EventNode::new("Event") + .with_child( + EventNode::new("System") + .with_child(EventNode::new("Computer").with_text("RING0IVY24-01")) + .with_child( + EventNode::new("Correlation") + .with_attribute("ActivityID", "{2f8b0c1e-0000-0000-0000-000000000000}"), + ), + ) + .with_child( + EventNode::new("EventData") + .with_child( + EventNode::new("Data") + .with_attribute("Name", "SubjectUserName") + .with_text("adam"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "LogonType") + .with_text("10"), + ), + ) + .with_child( + EventNode::new("UserData").with_child( + EventNode::new("EventInfo") + .with_child(EventNode::new("Username").with_text("TEST\\adam")), + ), + ) + } + + fn eval(expression: &str) -> Option { + ValuePath::parse(expression) + .expect("path parses") + .evaluate(&event()) + .map(|value| value.into_owned()) + } + + #[test] + fn resolves_named_event_data() { + assert_eq!( + eval(r#"/Event/EventData/Data[@Name="LogonType"]"#).as_deref(), + Some("10") + ); + } + + #[test] + fn resolves_single_quoted_predicate_value() { + assert_eq!( + eval("/Event/EventData/Data[@Name='SubjectUserName']").as_deref(), + Some("adam") + ); + } + + #[test] + fn bare_step_joins_repeated_elements_as_evtxecmd_does() { + // Verified against EvtxECmd on a real ESENT 326 record; see REPEATED_ELEMENT_SEPARATOR. + assert_eq!(eval("/Event/EventData/Data").as_deref(), Some("adam, 10")); + } + + #[test] + fn bare_step_with_a_single_match_returns_that_element_untouched() { + let single = EventNode::new("Event").with_child( + EventNode::new("EventData").with_child(EventNode::new("Data").with_text("RunOnceEx")), + ); + let path = ValuePath::parse("/Event/EventData/Data").expect("parses"); + assert_eq!(path.evaluate(&single).as_deref(), Some("RunOnceEx")); + } + + #[test] + fn joining_preserves_position_for_an_element_with_no_text() { + let gapped = EventNode::new("Event").with_child( + EventNode::new("EventData") + .with_child(EventNode::new("Data").with_text("first")) + .with_child(EventNode::new("Data")) + .with_child(EventNode::new("Data").with_text("third")), + ); + let path = ValuePath::parse("/Event/EventData/Data").expect("parses"); + assert_eq!(path.evaluate(&gapped).as_deref(), Some("first, , third")); + } + + #[test] + fn index_predicate_is_one_based() { + assert_eq!(eval("/Event/EventData/Data[1]").as_deref(), Some("adam")); + assert_eq!(eval("/Event/EventData/Data[2]").as_deref(), Some("10")); + assert_eq!(eval("/Event/EventData/Data[3]"), None); + } + + #[test] + fn resolves_nested_user_data() { + assert_eq!( + eval("/Event/UserData/EventInfo/Username").as_deref(), + Some("TEST\\adam") + ); + } + + #[test] + fn resolves_system_element_and_attribute() { + assert_eq!( + eval("/Event/System/Computer").as_deref(), + Some("RING0IVY24-01") + ); + assert_eq!( + eval("/Event/System/Correlation/@ActivityID").as_deref(), + Some("{2f8b0c1e-0000-0000-0000-000000000000}") + ); + } + + #[test] + fn missing_field_is_none_not_an_error() { + assert_eq!(eval(r#"/Event/EventData/Data[@Name="Absent"]"#), None); + assert_eq!(eval("/Event/EventData/Missing"), None); + } + + #[test] + fn container_without_text_resolves_to_none() { + assert_eq!(eval("/Event/EventData"), None); + } + + #[test] + fn rejects_relative_paths() { + assert!(matches!( + ValuePath::parse("Event/EventData"), + Err(PathError::NotAbsolute(_)) + )); + } + + #[test] + fn rejects_attribute_before_the_final_step() { + assert!(matches!( + ValuePath::parse("/Event/@Name/EventData"), + Err(PathError::MisplacedAttribute(_)) + )); + } + + #[test] + fn rejects_malformed_predicates() { + for expression in [ + "/Event/EventData/Data[@Name=\"unterminated", + "/Event/EventData/Data[@Name]", + "/Event/EventData/Data[abc]", + "/Event/EventData/Data[0]", + ] { + assert!( + matches!( + ValuePath::parse(expression), + Err(PathError::MalformedPredicate { .. }) + ), + "expected malformed predicate for {expression}" + ); + } + } + + #[test] + fn rejects_empty_paths() { + assert!(matches!(ValuePath::parse("/"), Err(PathError::Empty(_)))); + } + + #[test] + fn root_mismatch_resolves_to_none() { + let path = ValuePath::parse("/Other/EventData").expect("parses"); + assert_eq!(path.evaluate(&event()), None); + } + + #[test] + fn an_attribute_selector_reads_only_the_first_repeated_element() { + // Deliberately asymmetric with text content, which joins repeats with ", " to match + // EvtxECmd. Nothing pinned the attribute half, so a change making attributes join too + // would have passed the whole suite. + let event = EventNode::new("Event").with_child( + EventNode::new("EventData") + .with_child(EventNode::new("Data").with_attribute("Name", "first")) + .with_child(EventNode::new("Data").with_attribute("Name", "second")), + ); + + let path = ValuePath::parse("/Event/EventData/Data/@Name").expect("parses"); + assert_eq!(path.evaluate(&event).as_deref(), Some("first")); + } + + #[test] + fn a_doubled_separator_is_refused_rather_than_reinterpreted() { + // `/Event//EventData/Data` is a typo. Skipping the empty segment would read it as + // `/Event/EventData/Data` and apply a map the author did not write. + assert!(matches!( + ValuePath::parse("/Event//EventData/Data"), + Err(PathError::MalformedStep { .. }) + )); + } + + #[test] + fn an_attribute_with_no_name_is_refused() { + assert!(matches!( + ValuePath::parse("/Event/@"), + Err(PathError::MalformedStep { .. }) + )); + } + + #[test] + fn a_single_trailing_slash_names_the_same_element() { + // The one benign empty segment: `/Event/EventData/` selects what `/Event/EventData` does. + let with = ValuePath::parse("/Event/EventData/").expect("parses"); + let without = ValuePath::parse("/Event/EventData").expect("parses"); + assert_eq!(with, without); + } + + #[test] + fn a_bare_slash_is_still_an_empty_path() { + assert!(matches!(ValuePath::parse("/"), Err(PathError::Empty(_)))); + } +} diff --git a/crates/cmtraceopen-parser/src/lib.rs b/crates/cmtraceopen-parser/src/lib.rs index 2815a94fa..439f09f75 100644 --- a/crates/cmtraceopen-parser/src/lib.rs +++ b/crates/cmtraceopen-parser/src/lib.rs @@ -11,8 +11,13 @@ pub mod collector; pub mod dsregcmd; pub mod error_db; pub mod esp; +pub mod event_payload; +pub mod event_query; +pub mod eventmap; pub mod intune; pub mod models; pub mod parser; +pub mod provider; pub mod sccm; +pub mod unified_timeline; pub(crate) mod wire; diff --git a/crates/cmtraceopen-parser/src/models/log_entry.rs b/crates/cmtraceopen-parser/src/models/log_entry.rs index b7eeb515e..37db58ca6 100644 --- a/crates/cmtraceopen-parser/src/models/log_entry.rs +++ b/crates/cmtraceopen-parser/src/models/log_entry.rs @@ -4,9 +4,12 @@ use serde::{Deserialize, Serialize}; /// Maps directly to CMTrace's type field: 0=Success, 1=Info, 2=Warning, 3=Error. /// `Success` corresponds to CCM/PSADT `type="0"` — a completed operation that /// OneTrace renders with a green tick. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] pub enum Severity { Success, + /// Default: an unclassified line is informational, which is how every parser already treats + /// one it cannot rank. + #[default] Info, Warning, Error, @@ -24,13 +27,17 @@ pub enum EntryKind { } /// Which log format was detected/used to parse this entry. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] pub enum LogFormat { /// CCM/SCCM format: Ccm, /// Simple/legacy format: message$$ Simple, - /// Plain text (no structured format detected) + /// Plain text (no structured format detected). + /// + /// Default, because claiming a structured format that was never detected would be a stronger + /// assertion than the evidence supports. + #[default] Plain, /// Generic timestamped format (ISO 8601, slash-dates, syslog, time-only) Timestamped, @@ -154,7 +161,10 @@ pub struct ParserSelectionInfo { /// A single parsed log entry. /// Field names use camelCase for direct JSON serialization to TypeScript. -#[derive(Debug, Clone, Serialize, Deserialize)] +/// +/// `Default` exists so tests can construct one by naming only the fields under test; the struct +/// carries far too many fields to spell out every time. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct LogEntry { /// Sequential ID for stable row identity diff --git a/crates/cmtraceopen-parser/src/provider/mod.rs b/crates/cmtraceopen-parser/src/provider/mod.rs new file mode 100644 index 000000000..d74058aa3 --- /dev/null +++ b/crates/cmtraceopen-parser/src/provider/mod.rs @@ -0,0 +1,561 @@ +//! Rendering event descriptions from captured provider metadata. +//! +//! An event on disk carries values, not sentences. The sentence lives in the provider's message +//! table, which is why opening someone else's `.evtx` on your own machine shows raw `EventData` +//! and no description: the provider is not registered there. On macOS or Linux there is no +//! provider registry at all, so the problem is total rather than partial. +//! +//! EventLogExpert solves this by capturing provider metadata into a portable database. This module +//! is the rendering half of that: it takes already-deserialized metadata plus an event's insertion +//! strings and produces the description. It is pure, so it works identically on every platform, +//! which is the entire point. Reading the database file is the host layer's job. +//! +//! The format was reverse engineered from a real database built on Windows 11; the full spec is in +//! issue #539. + +use std::collections::BTreeMap; + +use serde::ser::SerializeSeq; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +/// Reinterprets a signed integer as unsigned, preserving the bit pattern. +/// +/// The metadata is serialized by a .NET tool, which writes `long` and `int`. A keyword mask with +/// the top bit set therefore appears as a negative number: the reserved keyword +/// `0x8000000000000000` is written as `-9223372036854775808`. Deserializing straight into `u64` +/// rejects those, which in practice meant the Microsoft-Windows-DeviceManagement provider, among +/// many others, failed to load at all. +fn signed_as_u64<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + Ok(i64::deserialize(deserializer)? as u64) +} + +fn signed_as_u64_vec<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + Ok(Vec::::deserialize(deserializer)? + .into_iter() + .map(|value| value as u64) + .collect()) +} + +/// Writes an unsigned value back in the signed form the source used. +/// +/// Needed because these types are public and derive `Serialize`. Without it a value round-trips +/// asymmetrically: `-9223372036854775808` deserializes to `0x8000000000000000`, serializes as +/// `9223372036854775808`, and then fails to deserialize again because the reader expects `i64`. +/// Anything that persisted or forwarded this metadata could not read its own output back. +fn u64_as_signed(value: &u64, serializer: S) -> Result { + serializer.serialize_i64(*value as i64) +} + +fn u64_vec_as_signed(values: &[u64], serializer: S) -> Result { + let mut sequence = serializer.serialize_seq(Some(values.len()))?; + for value in values { + sequence.serialize_element(&(*value as i64))?; + } + sequence.end() +} + +fn u32_as_signed(value: &u32, serializer: S) -> Result { + serializer.serialize_i64(*value as i32 as i64) +} + +/// Reinterprets a signed integer as an unsigned 32-bit value. +/// +/// Message identifiers above `0x7FFFFFFF` are written negative for the same reason. +fn signed_as_u32<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + Ok(i64::deserialize(deserializer)? as u32) +} + +/// One event definition from a provider's manifest. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, Default)] +#[serde(rename_all = "PascalCase")] +pub struct ProviderEvent { + /// Message template with `%1`-style insertion points. + #[serde(default)] + pub description: Option, + /// Event ID. + pub id: u32, + /// Version of this event's definition. Providers can define several versions of one ID. + #[serde(default)] + pub version: u32, + /// Channel the event belongs to. + #[serde(default)] + pub log_name: Option, + /// Level value. + #[serde(default)] + pub level: Option, + /// Task value. + #[serde(default)] + pub task: Option, + /// Opcode value. + #[serde(default)] + pub opcode: Option, + /// Keyword bitmask values. + #[serde( + default, + deserialize_with = "signed_as_u64_vec", + serialize_with = "u64_vec_as_signed" + )] + pub keywords: Vec, + /// The manifest template, which declares each field's name and type. + #[serde(default)] + pub template: Option, +} + +/// An entry from the provider's message table. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, Default)] +#[serde(rename_all = "PascalCase")] +pub struct ProviderMessage { + /// Full message identifier as the provider declares it. + #[serde( + default, + deserialize_with = "signed_as_u64", + serialize_with = "u64_as_signed" + )] + pub raw_id: u64, + /// Low bits of `raw_id`, which is what most references use. + #[serde( + default, + deserialize_with = "signed_as_u32", + serialize_with = "u32_as_signed" + )] + pub short_id: u32, + /// The message text. + #[serde(default)] + pub text: Option, +} + +/// Everything captured about one provider. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, Default)] +#[serde(rename_all = "PascalCase")] +pub struct ProviderMetadata { + /// Provider name, matching `System/Provider/@Name`. + #[serde(default)] + pub provider_name: String, + /// Event definitions. + #[serde(default)] + pub events: Vec, + /// Message table. + #[serde(default)] + pub messages: Vec, + /// Task value to name. + #[serde(default)] + pub tasks: BTreeMap, + /// Keyword bit to name. + #[serde(default)] + pub keywords: BTreeMap, + /// Opcode value to name. + #[serde(default)] + pub opcodes: BTreeMap, + /// Windows build the metadata was captured from, so a mismatch is visible rather than assumed. + #[serde(default)] + pub source_os_build: Option, +} + +impl ProviderMetadata { + /// Finds the definition for `event_id`, preferring an exact `version` match. + /// + /// Providers legitimately define several versions of one ID. Picking the wrong one renders a + /// description whose insertion points do not line up with the event's fields, which reads as + /// plausible but wrong text, so the exact version wins and the highest known version is only a + /// fallback. + pub fn event(&self, event_id: u32, version: Option) -> Option<&ProviderEvent> { + let candidates = self.events.iter().filter(|event| event.id == event_id); + if let Some(version) = version { + if let Some(exact) = candidates.clone().find(|event| event.version == version) { + return Some(exact); + } + } + candidates.max_by_key(|event| event.version) + } + + /// Resolves a task value to its name. + pub fn task_name(&self, task: u32) -> Option<&str> { + self.tasks.get(&task.to_string()).map(String::as_str) + } + + /// Resolves an opcode value to its name. + pub fn opcode_name(&self, opcode: u32) -> Option<&str> { + self.opcodes.get(&opcode.to_string()).map(String::as_str) + } + + /// Resolves a keyword bitmask to the names of the bits that are set. + /// + /// Only bits the provider declares are named. Undeclared bits are ignored rather than reported + /// as unknown keywords, because the reserved high bits are set by the system on most events. + pub fn keyword_names(&self, mask: u64) -> Vec<&str> { + // Sorted by bit value, not by key. The map is keyed by the decimal bit as a string, so its + // own order is lexicographic: "1", "16", "2", "32", "4". Returning that would put the names + // in an order matching neither the mask nor the provider's manifest, which reads as + // meaningful when it is an artefact of string comparison. + let mut matched: Vec<(u64, &str)> = self + .keywords + .iter() + .filter_map(|(raw_bit, name)| { + let bit = raw_bit.parse::().ok()?; + (bit != 0 && mask & bit == bit).then_some((bit, name.as_str())) + }) + .collect(); + matched.sort_by_key(|(bit, _)| *bit); + matched.into_iter().map(|(_, name)| name).collect() + } +} + +/// The outcome of rendering a description. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RenderedDescription { + /// The rendered text. + pub text: String, + /// Insertion numbers the template referenced but the event did not supply. + /// + /// Non-empty means `text` still contains those `%n` markers. They are left visible rather than + /// blanked, so a partially rendered description cannot be mistaken for a complete one. + pub missing_insertions: Vec, +} + +impl RenderedDescription { + /// True when every insertion point was filled. + pub fn is_complete(&self) -> bool { + self.missing_insertions.is_empty() + } +} + +/// Renders a provider message template against an event's insertion strings. +/// +/// `insertions` are the event's `EventData` values in document order, so `%1` is the first. +/// +/// Windows message syntax is much larger than this; only the parts that appear in captured event +/// descriptions are handled. `%%` is a literal percent and `%n` selects an insertion. Anything else +/// is passed through untouched rather than guessed at, because inventing a rendering is worse than +/// showing the provider's raw text. +pub fn render_description(template: &str, insertions: &[String]) -> RenderedDescription { + let mut text = String::with_capacity(template.len()); + let mut missing = Vec::new(); + let mut chars = template.char_indices().peekable(); + + while let Some((_, character)) = chars.next() { + if character != '%' { + text.push(character); + continue; + } + + match chars.peek().map(|(_, c)| *c) { + // "%%" is an escaped percent sign. + Some('%') => { + chars.next(); + text.push('%'); + } + Some(digit) if digit.is_ascii_digit() => { + let mut number = String::new(); + while let Some((_, c)) = chars.peek() { + if c.is_ascii_digit() { + number.push(*c); + chars.next(); + } else { + break; + } + } + let index: u32 = number.parse().unwrap_or(0); + match index + .checked_sub(1) + .and_then(|zero_based| insertions.get(zero_based as usize)) + { + Some(value) => text.push_str(value), + None => { + text.push('%'); + text.push_str(&number); + if !missing.contains(&index) { + missing.push(index); + } + } + } + } + // Not an insertion point, so the percent is literal content. + _ => text.push('%'), + } + } + + RenderedDescription { + text, + missing_insertions: missing, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn insertions(values: &[&str]) -> Vec { + values.iter().map(|v| v.to_string()).collect() + } + + #[test] + fn renders_a_real_mdm_description() { + // Verbatim from the DeviceManagement-Enterprise-Diagnostics-Provider metadata captured on + // Windows 11, event id 2. + let template = "MDM Enroll: Certificate policy create message failed. Result: (%1)."; + let rendered = render_description(template, &insertions(&["0x80180005"])); + assert_eq!( + rendered.text, + "MDM Enroll: Certificate policy create message failed. Result: (0x80180005)." + ); + assert!(rendered.is_complete()); + } + + #[test] + fn insertions_are_one_based_and_ordered() { + let rendered = render_description("%1 then %2 then %3", &insertions(&["a", "b", "c"])); + assert_eq!(rendered.text, "a then b then c"); + } + + #[test] + fn a_two_digit_insertion_is_not_read_as_two_single_digit_ones() { + // "%10" must select the tenth value, not the first followed by a literal zero. + let values = insertions(&["1", "2", "3", "4", "5", "6", "7", "8", "9", "TENTH"]); + assert_eq!(render_description("%10", &values).text, "TENTH"); + } + + #[test] + fn a_double_percent_is_a_literal_percent() { + let rendered = render_description("100%% complete", &[]); + assert_eq!(rendered.text, "100% complete"); + assert!(rendered.is_complete()); + } + + #[test] + fn a_missing_insertion_stays_visible_and_is_reported() { + let rendered = render_description("value is %1 and %2", &insertions(&["only-one"])); + assert_eq!(rendered.text, "value is only-one and %2"); + assert_eq!(rendered.missing_insertions, vec![2]); + assert!(!rendered.is_complete()); + } + + #[test] + fn a_percent_that_is_not_an_insertion_is_left_alone() { + let rendered = render_description("50% of %1", &insertions(&["disk"])); + assert_eq!(rendered.text, "50% of disk"); + assert!(rendered.is_complete()); + } + + #[test] + fn a_trailing_percent_does_not_panic() { + assert_eq!(render_description("done %", &[]).text, "done %"); + } + + #[test] + fn insertion_zero_is_reported_rather_than_wrapping() { + // %0 has no meaning as an insertion; treating it as index -1 would panic or wrap. + let rendered = render_description("%0", &insertions(&["a"])); + assert_eq!(rendered.text, "%0"); + assert_eq!(rendered.missing_insertions, vec![0]); + } + + fn metadata() -> ProviderMetadata { + ProviderMetadata { + provider_name: "Test-Provider".into(), + events: vec![ + ProviderEvent { + id: 100, + version: 0, + description: Some("v0 %1".into()), + ..Default::default() + }, + ProviderEvent { + id: 100, + version: 1, + description: Some("v1 %1".into()), + ..Default::default() + }, + ], + tasks: BTreeMap::from([("1".into(), "Enrollment".into())]), + opcodes: BTreeMap::from([("11".into(), "Start".into())]), + keywords: BTreeMap::from([ + ("1".into(), "Error".into()), + ("2".into(), "Debug".into()), + ("4".into(), "Trace".into()), + ]), + ..Default::default() + } + } + + #[test] + fn an_exact_version_match_wins() { + let meta = metadata(); + assert_eq!( + meta.event(100, Some(0)) + .and_then(|e| e.description.as_deref()), + Some("v0 %1") + ); + } + + #[test] + fn an_unknown_version_falls_back_to_the_highest_known() { + // Better a definition from a newer manifest than none, but never a silently wrong one when + // the exact version is available. + let meta = metadata(); + assert_eq!( + meta.event(100, Some(7)) + .and_then(|e| e.description.as_deref()), + Some("v1 %1") + ); + assert_eq!( + meta.event(100, None).and_then(|e| e.description.as_deref()), + Some("v1 %1") + ); + } + + #[test] + fn an_unknown_event_id_resolves_to_nothing() { + assert!(metadata().event(999, None).is_none()); + } + + #[test] + fn task_and_opcode_names_resolve() { + let meta = metadata(); + assert_eq!(meta.task_name(1), Some("Enrollment")); + assert_eq!(meta.task_name(2), None); + assert_eq!(meta.opcode_name(11), Some("Start")); + } + + #[test] + fn keyword_names_report_only_declared_bits_that_are_set() { + let meta = metadata(); + assert_eq!(meta.keyword_names(0b101), vec!["Error", "Trace"]); + assert_eq!(meta.keyword_names(0), Vec::<&str>::new()); + } + + #[test] + fn undeclared_keyword_bits_are_ignored_rather_than_reported_as_unknown() { + // Windows sets reserved high bits on most events; surfacing those as unknown keywords + // would make almost every event look anomalous. + let meta = metadata(); + assert_eq!(meta.keyword_names(0x8000_0000_0000_0001), vec!["Error"]); + } + + #[test] + fn a_high_bit_keyword_written_as_a_negative_number_round_trips() { + // The metadata is written by a .NET tool, so 0x8000000000000000 appears as + // -9223372036854775808. Rejecting that made whole providers fail to load. + let json = r#"{"Id":1,"Keywords":[-9223372036854775808,576460752303423488]}"#; + let event: ProviderEvent = serde_json::from_str(json).expect("deserializes"); + assert_eq!(event.keywords[0], 0x8000_0000_0000_0000); + assert_eq!(event.keywords[1], 576_460_752_303_423_488); + } + + #[test] + fn a_negative_message_id_is_reinterpreted_rather_than_rejected() { + let json = r#"{"RawId":-2147221478,"ShortId":-2147221478,"Text":"x"}"#; + let message: ProviderMessage = serde_json::from_str(json).expect("deserializes"); + assert_eq!(message.raw_id, (-2_147_221_478_i64) as u64); + assert_eq!(message.short_id, (-2_147_221_478_i64) as u32); + } + + #[test] + fn ordinary_positive_values_are_unaffected() { + let json = r#"{"RawId":1342177282,"ShortId":2,"Text":"Error"}"#; + let message: ProviderMessage = serde_json::from_str(json).expect("deserializes"); + assert_eq!(message.raw_id, 1_342_177_282); + assert_eq!(message.short_id, 2); + } + + #[test] + fn a_keyword_mask_with_the_reserved_high_bit_still_resolves_declared_names() { + let meta = metadata(); + assert_eq!( + meta.keyword_names(0x8000_0000_0000_0000_u64 | 1), + vec!["Error"] + ); + } + + #[test] + fn metadata_deserializes_from_the_captured_shape() { + // Field names as they appear in a real provider database row. + let json = r#"{ + "ProviderName": "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider", + "Events": [{ + "Description": "MDM Enroll: failed. Result: (%1).", + "Id": 2, "Version": 0, "Level": 2, "Task": 0, "Opcode": 0, + "Keywords": [576460752303423488], + "LogName": "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider/Enrollment", + "Template": "" + }], + "Tasks": {"1": "None"}, + "Keywords": {"1": "Error"}, + "SourceOsBuild": 26200 + }"#; + let meta: ProviderMetadata = serde_json::from_str(json).expect("deserializes"); + assert_eq!(meta.events.len(), 1); + assert_eq!(meta.events[0].id, 2); + assert_eq!(meta.events[0].keywords, vec![576_460_752_303_423_488]); + assert_eq!(meta.source_os_build, Some(26200)); + assert_eq!(meta.task_name(1), Some("None")); + } + + #[test] + fn keyword_names_come_back_in_bit_order_not_key_order() { + // The map is keyed by the decimal bit as a string, so its own order is "1", "16", "2", + // "32", "4". Returning that reads as meaningful when it is an artefact of string sorting. + let mut keywords = std::collections::BTreeMap::new(); + for (bit, name) in [ + ("1", "Startup"), + ("2", "Shutdown"), + ("4", "Network"), + ("16", "Disk"), + ("32", "Memory"), + ] { + keywords.insert(bit.to_string(), name.to_string()); + } + let metadata = ProviderMetadata { + keywords, + ..Default::default() + }; + + assert_eq!( + metadata.keyword_names(0b11_0111), + vec!["Startup", "Shutdown", "Network", "Disk", "Memory"] + ); + } + + #[test] + fn only_the_bits_present_in_the_mask_are_named() { + let mut keywords = std::collections::BTreeMap::new(); + keywords.insert("1".to_string(), "Startup".to_string()); + keywords.insert("16".to_string(), "Disk".to_string()); + keywords.insert("32".to_string(), "Memory".to_string()); + let metadata = ProviderMetadata { + keywords, + ..Default::default() + }; + + assert_eq!(metadata.keyword_names(0b10_0001), vec!["Startup", "Memory"]); + assert!(metadata.keyword_names(0).is_empty()); + } + + #[test] + fn a_top_bit_keyword_survives_a_serialization_round_trip() { + // The .NET source writes these signed. Deserializing accepted that and serializing wrote + // the unsigned form, so the type could not read its own output back: anything that + // persisted or forwarded provider metadata broke on the reserved keyword alone. + let json = r#"{"Id":1,"Version":0,"Keywords":[-9223372036854775808]}"#; + let event: ProviderEvent = serde_json::from_str(json).expect("deserializes"); + assert_eq!(event.keywords, vec![0x8000_0000_0000_0000]); + + let written = serde_json::to_string(&event).expect("serializes"); + assert!( + written.contains("-9223372036854775808"), + "the signed form the source used must be preserved: {written}" + ); + + let again: ProviderEvent = serde_json::from_str(&written).expect("re-reads its own output"); + assert_eq!(again, event); + } + + #[test] + fn an_ordinary_keyword_is_unchanged_by_the_round_trip() { + let json = r#"{"Id":1,"Version":0,"Keywords":[16]}"#; + let event: ProviderEvent = serde_json::from_str(json).expect("deserializes"); + let written = serde_json::to_string(&event).expect("serializes"); + assert!(written.contains("[16]"), "{written}"); + let again: ProviderEvent = serde_json::from_str(&written).expect("re-reads"); + assert_eq!(again.keywords, vec![16]); + } +} diff --git a/crates/cmtraceopen-parser/src/unified_timeline/mod.rs b/crates/cmtraceopen-parser/src/unified_timeline/mod.rs new file mode 100644 index 000000000..286e3836f --- /dev/null +++ b/crates/cmtraceopen-parser/src/unified_timeline/mod.rs @@ -0,0 +1,455 @@ +//! Merging Windows events and parsed text logs into one chronological view. +//! +//! Every event viewer surveyed for issue #539 is events-only, and every log viewer is text-only. +//! Nobody puts them side by side, which is the one thing that actually explains a failure: the +//! `DeviceManagement-Enterprise-Diagnostics-Provider` event says enrollment failed with an HRESULT, +//! and the `IntuneManagementExtension.log` line thirty seconds earlier says why. +//! +//! This module owns the merge. It is pure and holds no knowledge of where items came from, so the +//! event side can live in the host layer while the log side comes straight from +//! [`LogEntry`](crate::models::log_entry::LogEntry). +//! +//! The hard part is not sorting. It is refusing to place things that cannot honestly be placed: +//! an entry with no timestamp has no position on a timeline, and putting it at the epoch or at the +//! previous entry's time would invent a sequence the evidence does not support. + +use serde::{Deserialize, Serialize}; + +use crate::models::log_entry::{LogEntry, Severity}; + +/// Severity normalized across both sides of the merge. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum TimelineSeverity { + /// Diagnostic detail, from a verbose event. Text logs have no equivalent. + Verbose, + /// Ordinary progress. The default, since a line carrying no severity is not a problem. + #[default] + Info, + /// Something the source flagged as worth noticing but did not treat as a failure. + Warning, + /// A failure the source reported. The highest a text log reaches. + Error, + /// Only events carry this; text logs top out at error. + Critical, +} + +impl TimelineSeverity { + /// Maps a text log severity. + /// + /// `Success` maps to `Info` rather than gaining a level of its own: on a merged timeline it is + /// an ordinary informational line, and a separate rank would sort it away from the events it + /// sits between. + pub fn from_log(severity: Severity) -> Self { + match severity { + Severity::Error => Self::Error, + Severity::Warning => Self::Warning, + Severity::Info | Severity::Success => Self::Info, + } + } + + /// Maps a Windows event level value as written in `System/Level`. + /// + /// Level 0 means "not set", which providers use for events that are not classified. It maps to + /// `Info` because treating an unclassified event as critical would flood a severity filter. + pub fn from_event_level(level: u8) -> Self { + match level { + 1 => Self::Critical, + 2 => Self::Error, + 3 => Self::Warning, + 5 => Self::Verbose, + _ => Self::Info, + } + } +} + +/// Where a timeline item came from. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +// Growable: a third kind of source, such as a registry export or an ETW trace read directly, is a +// plausible addition. Marking it now keeps that a minor change. +#[non_exhaustive] +// rename_all covers the variant names; rename_all_fields is what camel-cases the fields inside a +// struct variant. Without the second one, event_id went over the wire as "event_id" while the +// timeline view reads origin.eventId, so every event row rendered undefined. +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase" +)] +pub enum TimelineOrigin { + /// A line from a parsed text log. + Log { + /// File the line came from, as the parser recorded it. + file: String, + /// Emitting component, when the format carries one. + component: Option, + /// 1-based line number, so the item can be traced back to the source. + line: u32, + }, + /// A Windows event. + Event { + /// Channel the event was read from, for example `Microsoft-Windows-DNSServer/Audit`. + channel: String, + /// Publisher that raised it, as the event's own `System` block names it. + provider: String, + /// Event ID, which identifies the event only in combination with the provider. + event_id: u32, + /// Record identifier within its channel. + record_id: u64, + }, +} + +/// One placed item. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TimelineItem { + /// Milliseconds since the Unix epoch. + pub timestamp_ms: i64, + /// How serious the source said this was, normalized across event levels and log levels. + pub severity: TimelineSeverity, + /// The rendered text, already whatever the source's own formatting produced. + pub message: String, + /// Which file or channel this came from, so a row on a merged timeline stays attributable. + pub origin: TimelineOrigin, +} + +/// Something that could not be placed, and why. +/// +/// Surfaced rather than dropped: a timeline that silently omits a third of a log file looks +/// complete and is not. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct UnplacedItem { + /// Where it came from, so an operator can go and look. + pub origin: TimelineOrigin, + /// Why it has no position. + pub reason: UnplacedReason, +} + +/// Why an item has no position on the timeline. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +// Growable: this build does not know every variant a newer schema will define. +// Marking it now keeps adding one a minor change; after the first release that +// exposes the type, adding the attribute is itself breaking. +#[non_exhaustive] +pub enum UnplacedReason { + /// The source carried no timestamp, or one the parser could not read. + /// + /// Common in continuation lines and in text logs whose first line is a header. + MissingTimestamp, +} + +/// A merged timeline plus everything that could not be placed on it. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct UnifiedTimeline { + /// Items in chronological order. + pub items: Vec, + /// Items with no honest position, in input order. + pub unplaced: Vec, +} + +impl UnifiedTimeline { + /// True when everything supplied was placed. + pub fn is_complete(&self) -> bool { + self.unplaced.is_empty() + } + + /// Inclusive time span covered, or `None` when nothing was placed. + /// + /// The bounds are computed rather than read off the ends. `merge` sorts, so reading the ends + /// would be correct on that path, but `items` is a public field and a value built directly can + /// hold items in any order. Reporting a span narrower than the data would understate what a + /// timeline actually covers. + pub fn span_ms(&self) -> Option<(i64, i64)> { + let mut stamps = self.items.iter().map(|item| item.timestamp_ms); + let first = stamps.next()?; + Some(stamps.fold((first, first), |(low, high), stamp| { + (low.min(stamp), high.max(stamp)) + })) + } +} + +/// Converts a parsed log entry, or reports why it cannot be placed. +pub fn from_log_entry(entry: &LogEntry) -> Result { + let origin = TimelineOrigin::Log { + file: entry + .source_file + .clone() + .unwrap_or_else(|| entry.file_path.clone()), + component: entry.component.clone(), + line: entry.line_number, + }; + + match entry.timestamp { + Some(timestamp_ms) => Ok(TimelineItem { + timestamp_ms, + severity: TimelineSeverity::from_log(entry.severity), + message: entry.message.clone(), + origin, + }), + None => Err(UnplacedItem { + origin, + reason: UnplacedReason::MissingTimestamp, + }), + } +} + +/// Merges already-converted items into one chronological timeline. +/// +/// Ordering is stable: items sharing a timestamp keep the order they were supplied in. That matters +/// because a text log and an event recorded in the same millisecond have no discoverable ordering +/// between them, and re-sorting on severity or source would invent one. +pub fn merge( + placed: impl IntoIterator, + unplaced: impl IntoIterator, +) -> UnifiedTimeline { + let mut items: Vec = placed.into_iter().collect(); + // sort_by_key is a stable sort, which is what preserves input order within a timestamp. + items.sort_by_key(|item| item.timestamp_ms); + UnifiedTimeline { + items, + unplaced: unplaced.into_iter().collect(), + } +} + +/// Converts and merges a slice of log entries, collecting the ones that cannot be placed. +pub fn from_log_entries(entries: &[LogEntry]) -> UnifiedTimeline { + let mut placed = Vec::new(); + let mut unplaced = Vec::new(); + for entry in entries { + match from_log_entry(entry) { + Ok(item) => placed.push(item), + Err(reason) => unplaced.push(reason), + } + } + merge(placed, unplaced) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn log_entry(timestamp: Option, message: &str, severity: Severity) -> LogEntry { + LogEntry { + id: 0, + line_number: 7, + message: message.to_string(), + component: Some("IME".to_string()), + timestamp, + severity, + file_path: "C:/logs/IntuneManagementExtension.log".to_string(), + ..LogEntry::default() + } + } + + fn event(timestamp_ms: i64, message: &str, severity: TimelineSeverity) -> TimelineItem { + TimelineItem { + timestamp_ms, + severity, + message: message.to_string(), + origin: TimelineOrigin::Event { + channel: "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider/Admin" + .to_string(), + provider: "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider" + .to_string(), + event_id: 76, + record_id: 1, + }, + } + } + + #[test] + fn events_and_log_lines_interleave_by_time() { + // The whole point: the event says enrollment failed, the log line before it says why. + let logs = [ + log_entry(Some(1_000), "Checking enrollment", Severity::Info), + log_entry(Some(3_000), "Token request rejected", Severity::Error), + ]; + let timeline = merge( + logs.iter() + .filter_map(|entry| from_log_entry(entry).ok()) + .chain([event(2_000, "MDM enroll failed", TimelineSeverity::Error)]), + [], + ); + + let messages: Vec<&str> = timeline.items.iter().map(|i| i.message.as_str()).collect(); + assert_eq!( + messages, + vec![ + "Checking enrollment", + "MDM enroll failed", + "Token request rejected" + ] + ); + } + + #[test] + fn an_entry_without_a_timestamp_is_reported_rather_than_placed() { + // Placing it at the epoch, or at the previous entry's time, would invent a sequence. + let entries = [ + log_entry(Some(1_000), "placed", Severity::Info), + log_entry(None, "continuation line", Severity::Info), + ]; + let timeline = from_log_entries(&entries); + + assert_eq!(timeline.items.len(), 1); + assert_eq!(timeline.unplaced.len(), 1); + assert_eq!( + timeline.unplaced[0].reason, + UnplacedReason::MissingTimestamp + ); + assert!(!timeline.is_complete()); + } + + #[test] + fn an_unplaced_item_still_says_where_it_came_from() { + let timeline = from_log_entries(&[log_entry(None, "orphan", Severity::Error)]); + match &timeline.unplaced[0].origin { + TimelineOrigin::Log { + file, + line, + component, + } => { + assert!(file.ends_with("IntuneManagementExtension.log")); + assert_eq!(*line, 7); + assert_eq!(component.as_deref(), Some("IME")); + } + other => panic!("expected a log origin, got {other:?}"), + } + } + + #[test] + fn items_sharing_a_timestamp_keep_their_input_order() { + // A log line and an event in the same millisecond have no discoverable ordering, so the + // merge must not invent one by sorting on severity or source. + let timeline = merge( + [ + event(5_000, "first supplied", TimelineSeverity::Error), + event(5_000, "second supplied", TimelineSeverity::Verbose), + event(5_000, "third supplied", TimelineSeverity::Critical), + ], + [], + ); + let messages: Vec<&str> = timeline.items.iter().map(|i| i.message.as_str()).collect(); + assert_eq!( + messages, + vec!["first supplied", "second supplied", "third supplied"] + ); + } + + #[test] + fn negative_timestamps_sort_before_the_epoch_rather_than_wrapping() { + let timeline = merge( + [ + event(10, "after", TimelineSeverity::Info), + event(-10, "before", TimelineSeverity::Info), + ], + [], + ); + assert_eq!(timeline.items[0].message, "before"); + } + + #[test] + fn the_span_covers_first_to_last() { + let timeline = merge( + [ + event(3_000, "c", TimelineSeverity::Info), + event(1_000, "a", TimelineSeverity::Info), + ], + [], + ); + assert_eq!(timeline.span_ms(), Some((1_000, 3_000))); + } + + #[test] + fn an_empty_timeline_has_no_span_rather_than_a_zero_one() { + let timeline = UnifiedTimeline::default(); + assert_eq!(timeline.span_ms(), None); + assert!(timeline.is_complete()); + } + + #[test] + fn log_success_is_informational_not_a_rank_of_its_own() { + // A separate rank would sort Success away from the events it sits between. + assert_eq!( + TimelineSeverity::from_log(Severity::Success), + TimelineSeverity::Info + ); + assert_eq!( + TimelineSeverity::from_log(Severity::Error), + TimelineSeverity::Error + ); + } + + #[test] + fn event_level_zero_is_informational_not_critical() { + // Level 0 means "not set". Ranking it critical would flood a severity filter. + assert_eq!( + TimelineSeverity::from_event_level(0), + TimelineSeverity::Info + ); + assert_eq!( + TimelineSeverity::from_event_level(1), + TimelineSeverity::Critical + ); + assert_eq!( + TimelineSeverity::from_event_level(5), + TimelineSeverity::Verbose + ); + // An undeclared level must not panic or invent a rank. + assert_eq!( + TimelineSeverity::from_event_level(9), + TimelineSeverity::Info + ); + } + + #[test] + fn severity_orders_from_verbose_to_critical() { + assert!(TimelineSeverity::Verbose < TimelineSeverity::Info); + assert!(TimelineSeverity::Error < TimelineSeverity::Critical); + } + + #[test] + fn a_log_entry_falls_back_to_its_file_path_when_no_source_file_is_recorded() { + let timeline = from_log_entries(&[log_entry(Some(1), "x", Severity::Info)]); + match &timeline.items[0].origin { + TimelineOrigin::Log { file, .. } => assert!(file.contains("IntuneManagementExtension")), + other => panic!("expected a log origin, got {other:?}"), + } + } + + #[test] + fn an_event_origin_serializes_with_the_keys_the_frontend_reads() { + // rename_all on an enum renames the variants, not the fields inside a struct variant, so + // this needs rename_all_fields. Without it event_id went over the wire as "event_id" while + // the timeline view reads origin.eventId, and every event row showed undefined. + let origin = TimelineOrigin::Event { + channel: "Application".into(), + provider: "ESENT".into(), + event_id: 326, + record_id: 42, + }; + let json = serde_json::to_value(&origin).expect("serializes"); + + assert_eq!(json["kind"], "event"); + assert_eq!(json["eventId"], 326); + assert_eq!(json["recordId"], 42); + assert!(json.get("event_id").is_none(), "{json}"); + assert!(json.get("record_id").is_none(), "{json}"); + } + + #[test] + fn a_log_origin_keeps_its_wire_keys() { + let origin = TimelineOrigin::Log { + file: "cmt.log".into(), + component: Some("Agent".into()), + line: 7, + }; + let json = serde_json::to_value(&origin).expect("serializes"); + assert_eq!(json["kind"], "log"); + assert_eq!(json["file"], "cmt.log"); + assert_eq!(json["line"], 7); + } +} diff --git a/crates/cmtraceopen-parser/tests/eventmap_corpus.rs b/crates/cmtraceopen-parser/tests/eventmap_corpus.rs new file mode 100644 index 000000000..bf35c0ba4 --- /dev/null +++ b/crates/cmtraceopen-parser/tests/eventmap_corpus.rs @@ -0,0 +1,245 @@ +//! The map engine driven by unmodified upstream EvtxECmd maps. +//! +//! These fixtures are real corpus (see `tests/fixtures/eventmap/README.md`), converted from YAML +//! to JSON without touching keys or values. The point is to prove the schema is implemented as +//! shipped rather than as imagined, so a regression here means the engine has drifted from the +//! community format. + +use cmtraceopen_parser::eventmap::{ + apply_map, EventMap, EventNode, MapProperty, MapRegistry, ValuePath, +}; + +const SHELL_CORE_9701: &str = include_str!("fixtures/eventmap/shell-core-9701.json"); +const SECURITY_4624: &str = include_str!("fixtures/eventmap/security-4624.json"); +const NTFS_146: &str = include_str!("fixtures/eventmap/ntfs-146-lookups.json"); + +fn load(raw: &str) -> EventMap { + serde_json::from_str(raw).expect("upstream map deserializes") +} + +/// Builds an `Event` whose `EventData` holds the given named `Data` elements. +fn event_with_named_data(fields: &[(&str, &str)]) -> EventNode { + let event_data = + fields + .iter() + .fold(EventNode::new("EventData"), |event_data, (name, value)| { + event_data.with_child( + EventNode::new("Data") + .with_attribute("Name", *name) + .with_text(*value), + ) + }); + EventNode::new("Event").with_child(event_data) +} + +#[test] +fn shell_core_9701_maps_unnamed_event_data() { + let map = load(SHELL_CORE_9701); + assert_eq!(map.event_id, 9701); + assert_eq!(map.channel, "Microsoft-Windows-Shell-Core/Operational"); + assert_eq!(map.author.as_deref(), Some("Troy Larson")); + + let event = EventNode::new("Event").with_child( + EventNode::new("EventData") + .with_child(EventNode::new("Data").with_text("RunOnceEx commands started")), + ); + + let mapped = apply_map(&map, &event); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("RunOnceEx commands started") + ); + assert!(mapped.invalid_paths.is_empty()); + assert!(mapped.values.iter().all(|value| value.is_complete())); +} + +#[test] +fn security_4624_fills_every_column_from_a_complete_event() { + let map = load(SECURITY_4624); + assert_eq!(map.maps.len(), 8); + + let event = event_with_named_data(&[ + ("SubjectDomainName", "TEST"), + ("SubjectUserName", "adam"), + ("IpAddress", "192.168.16.103"), + ("WorkstationName", "RING0IVY24-01"), + ("TargetDomainName", "TEST"), + ("TargetUserName", "svc-collector"), + ("LogonType", "10"), + ("TargetLogonId", "0x3e7"), + ("AuthenticationPackageName", "Negotiate"), + ("LogonProcessName", "Advapi"), + ("ProcessName", "C:\\Windows\\System32\\svchost.exe"), + ]); + + let mapped = apply_map(&map, &event); + + assert_eq!(mapped.value_for(&MapProperty::UserName), Some("TEST\\adam")); + assert_eq!( + mapped.value_for(&MapProperty::RemoteHost), + Some("RING0IVY24-01 (192.168.16.103)") + ); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("Target: TEST\\svc-collector") + ); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(2)), + Some("LogonType 10") + ); + assert_eq!( + mapped.value_for(&MapProperty::ExecutableInfo), + Some("C:\\Windows\\System32\\svchost.exe") + ); + assert!( + mapped.values.iter().all(|value| value.is_complete()), + "every column should resolve from a complete event" + ); +} + +#[test] +fn security_4624_reports_absent_fields_instead_of_blanking_them() { + let map = load(SECURITY_4624); + + // A real 4624 from a network logon carries no WorkstationName. + let event = event_with_named_data(&[ + ("SubjectDomainName", "TEST"), + ("SubjectUserName", "adam"), + ("IpAddress", "192.168.16.103"), + ]); + + let mapped = apply_map(&map, &event); + + assert_eq!(mapped.value_for(&MapProperty::UserName), Some("TEST\\adam")); + + let remote_host = mapped + .values + .iter() + .find(|value| value.property == MapProperty::RemoteHost) + .expect("RemoteHost column exists"); + assert_eq!(remote_host.unresolved, vec!["workstation".to_string()]); + assert!(!remote_host.is_complete()); + assert!( + remote_host.text.contains("192.168.16.103"), + "the field that did resolve is still shown: {}", + remote_host.text + ); + assert!( + remote_host.text.contains("%workstation%"), + "the missing field stays visibly unresolved rather than rendering as blank: {}", + remote_host.text + ); +} + +#[test] +fn ntfs_146_applies_its_lookup_table_and_default() { + let map = load(NTFS_146); + let lookup = map.lookup_for("BusType").expect("BusType lookup present"); + assert_eq!(lookup.default.as_deref(), Some("Unknown code")); + + // Asserted on the column the binding targets, not on "some column somewhere". `any` over every + // value passed if the string turned up anywhere, so it did not establish that the BusType + // binding itself was translated, and "does not contain : 7" was only an indirect proxy for + // the raw code having been replaced. + let usb = apply_map( + &map, + &event_with_named_data(&[("VolumeName", "C:"), ("BusType", "7")]), + ); + let translated = usb + .values + .iter() + .find(|value| value.text.contains("USB")) + .unwrap_or_else(|| { + panic!( + "no value rendered the BusType translation: {:?}", + usb.values.iter().map(|v| &v.text).collect::>() + ) + }); + assert!( + !translated.text.contains('7'), + "the raw code should be gone once translated: {}", + translated.text + ); + + let unknown = apply_map( + &map, + &event_with_named_data(&[("VolumeName", "C:"), ("BusType", "255")]), + ); + let defaulted = unknown + .values + .iter() + .find(|value| value.text.contains("Unknown code")) + .unwrap_or_else(|| { + panic!( + "an out-of-table code should fall back to the lookup default: {:?}", + unknown.values.iter().map(|v| &v.text).collect::>() + ) + }); + assert!( + !defaulted.text.contains("255"), + "the default replaces the code rather than appending to it: {}", + defaulted.text + ); +} + +#[test] +fn a_registry_resolves_each_fixture_by_its_own_identity() { + let mut registry = MapRegistry::new(); + for raw in [SHELL_CORE_9701, SECURITY_4624, NTFS_146] { + registry.insert(load(raw)); + } + assert_eq!(registry.len(), 3); + + let mapped = registry + .apply( + "Microsoft-Windows-Shell-Core/Operational", + "Microsoft-Windows-Shell-Core", + 9701, + &EventNode::new("Event").with_child( + EventNode::new("EventData").with_child(EventNode::new("Data").with_text("cmd.exe")), + ), + ) + .expect("registered map is found"); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("cmd.exe") + ); + + // Right channel and provider, wrong event: no map, and no guessing. + assert!(registry + .apply( + "Microsoft-Windows-Shell-Core/Operational", + "Microsoft-Windows-Shell-Core", + 9702, + &EventNode::new("Event") + ) + .is_none()); +} + +#[test] +fn every_fixture_parses_with_no_malformed_paths() { + // Every expression is parsed directly rather than inferred from apply_map. Applying a map + // skips any binding whose %Name% is absent from its template, and a skipped binding never + // reaches the parser, so a malformed expression on one would leave invalid_paths empty and + // this test would pass while proving nothing about that binding. + let mut checked = 0usize; + for raw in [SHELL_CORE_9701, SECURITY_4624, NTFS_146] { + let map = load(raw); + for entry in &map.maps { + for binding in entry.bindings() { + assert!( + ValuePath::parse(&binding.value).is_ok(), + "upstream map {} binding {} has an expression this engine cannot parse: {}", + map.event_id, + binding.name, + binding.value + ); + checked += 1; + } + } + } + assert!( + checked >= 3, + "the fixtures should contribute several bindings, saw {checked}" + ); +} diff --git a/crates/cmtraceopen-parser/tests/fixtures/eventmap/README.md b/crates/cmtraceopen-parser/tests/fixtures/eventmap/README.md new file mode 100644 index 000000000..d7a24bc67 --- /dev/null +++ b/crates/cmtraceopen-parser/tests/fixtures/eventmap/README.md @@ -0,0 +1,15 @@ +# EvtxECmd map fixtures + +These are unmodified EvtxECmd maps from [EricZimmerman/evtx](https://github.com/EricZimmerman/evtx) +(MIT licensed), converted from YAML to JSON. Only the serialization format changed; the keys, +values, and structure are preserved exactly as upstream wrote them. The bytes necessarily differ, +since that is what the conversion does. + +They are vendored so the map engine is tested against the real schema rather than invented +examples, per the repository rule that fixtures must anchor to real corpus. + +| Fixture | Upstream file | Why it is here | +|---|---|---| +| `shell-core-9701.json` | `Microsoft-Windows-Shell-Core-Operational_Microsoft-Windows-Shell-Core_9701.map` | Simplest shape: one entry, bare `/Event/EventData/Data`. RunOnceEx during Autopilot OOBE. | +| `security-4624.json` | `Security_Microsoft-Windows-Security-Auditing_4624.map` | Multiple bindings per entry, multi-placeholder templates, `UserName` and `RemoteHost` targets. | +| `ntfs-146-lookups.json` | `Microsoft-Windows-Ntfs-Operational_Microsoft-Windows-Ntfs_146.map` | Exercises `Lookups` with a `Default` fallback. | diff --git a/crates/cmtraceopen-parser/tests/fixtures/eventmap/ntfs-146-lookups.json b/crates/cmtraceopen-parser/tests/fixtures/eventmap/ntfs-146-lookups.json new file mode 100644 index 000000000..8db122b40 --- /dev/null +++ b/crates/cmtraceopen-parser/tests/fixtures/eventmap/ntfs-146-lookups.json @@ -0,0 +1,96 @@ +{ + "Author": "Hyun Yi @hyuunnn", + "Description": "IO Connection", + "EventId": 146, + "Channel": "Microsoft-Windows-Ntfs/Operational", + "Provider": "Microsoft-Windows-Ntfs", + "Maps": [ + { + "Property": "PayloadData1", + "PropertyValue": "VolumeName: %VolumeName%", + "Values": [ + { + "Name": "VolumeName", + "Value": "/Event/EventData/Data[@Name=\"VolumeName\"]" + } + ] + }, + { + "Property": "PayloadData2", + "PropertyValue": "IsBootVolume: %IsBootVolume%", + "Values": [ + { + "Name": "IsBootVolume", + "Value": "/Event/EventData/Data[@Name=\"IsBootVolume\"]" + } + ] + }, + { + "Property": "PayloadData3", + "PropertyValue": "VendorId: %VendorId%", + "Values": [ + { + "Name": "VendorId", + "Value": "/Event/EventData/Data[@Name=\"VendorId\"]" + } + ] + }, + { + "Property": "PayloadData4", + "PropertyValue": "ProductId: %ProductId%", + "Values": [ + { + "Name": "ProductId", + "Value": "/Event/EventData/Data[@Name=\"ProductId\"]" + } + ] + }, + { + "Property": "PayloadData5", + "PropertyValue": "DeviceSerialNumber: %DeviceSerialNumber%", + "Values": [ + { + "Name": "DeviceSerialNumber", + "Value": "/Event/EventData/Data[@Name=\"DeviceSerialNumber\"]" + } + ] + }, + { + "Property": "PayloadData6", + "PropertyValue": "BusType: %BusType%", + "Values": [ + { + "Name": "BusType", + "Value": "/Event/EventData/Data[@Name=\"BusType\"]" + } + ] + } + ], + "Lookups": [ + { + "Name": "BusType", + "Default": "Unknown code", + "Values": { + "0": "The bus type is unknown.", + "1": "SCSI", + "2": "ATAPI", + "3": "ATA", + "4": "IEEE 1394", + "5": "SSA", + "6": "Fibre Channel", + "7": "USB", + "8": "RAID", + "9": "iSCSI", + "10": "Serial Attached SCSI (SAS)", + "11": "Serial ATA (SATA)", + "12": "Secure Digital (SD)", + "13": "Multimedia Card (MMC)", + "14": "This value is reserved for system use.", + "15": "File-Backed Virtual", + "16": "Storage Spaces", + "17": "NVMe", + "18": "This value is reserved for system use." + } + } + ] +} diff --git a/crates/cmtraceopen-parser/tests/fixtures/eventmap/security-4624.json b/crates/cmtraceopen-parser/tests/fixtures/eventmap/security-4624.json new file mode 100644 index 000000000..af92122a3 --- /dev/null +++ b/crates/cmtraceopen-parser/tests/fixtures/eventmap/security-4624.json @@ -0,0 +1,101 @@ +{ + "Author": "Eric Zimmerman saericzimmerman@gmail.com", + "Description": "Successful logon", + "EventId": 4624, + "Channel": "Security", + "Provider": "Microsoft-Windows-Security-Auditing", + "Maps": [ + { + "Property": "UserName", + "PropertyValue": "%domain%\\%user%", + "Values": [ + { + "Name": "domain", + "Value": "/Event/EventData/Data[@Name=\"SubjectDomainName\"]" + }, + { + "Name": "user", + "Value": "/Event/EventData/Data[@Name=\"SubjectUserName\"]" + } + ] + }, + { + "Property": "RemoteHost", + "PropertyValue": "%workstation% (%ipAddress%)", + "Values": [ + { + "Name": "ipAddress", + "Value": "/Event/EventData/Data[@Name=\"IpAddress\"]" + }, + { + "Name": "workstation", + "Value": "/Event/EventData/Data[@Name=\"WorkstationName\"]" + } + ] + }, + { + "Property": "PayloadData1", + "PropertyValue": "Target: %TargetDomainName%\\%TargetUserName%", + "Values": [ + { + "Name": "TargetDomainName", + "Value": "/Event/EventData/Data[@Name=\"TargetDomainName\"]" + }, + { + "Name": "TargetUserName", + "Value": "/Event/EventData/Data[@Name=\"TargetUserName\"]" + } + ] + }, + { + "Property": "PayloadData2", + "PropertyValue": "LogonType %LogonType%", + "Values": [ + { + "Name": "LogonType", + "Value": "/Event/EventData/Data[@Name=\"LogonType\"]" + } + ] + }, + { + "Property": "PayloadData3", + "PropertyValue": "LogonId: %TargetLogonId%", + "Values": [ + { + "Name": "TargetLogonId", + "Value": "/Event/EventData/Data[@Name=\"TargetLogonId\"]" + } + ] + }, + { + "Property": "PayloadData4", + "PropertyValue": "AuthenticationPackageName: %AuthenticationPackageName%", + "Values": [ + { + "Name": "AuthenticationPackageName", + "Value": "/Event/EventData/Data[@Name=\"AuthenticationPackageName\"]" + } + ] + }, + { + "Property": "PayloadData5", + "PropertyValue": "LogonProcessName: %LogonProcessName%", + "Values": [ + { + "Name": "LogonProcessName", + "Value": "/Event/EventData/Data[@Name=\"LogonProcessName\"]" + } + ] + }, + { + "Property": "ExecutableInfo", + "PropertyValue": "%ProcessName%", + "Values": [ + { + "Name": "ProcessName", + "Value": "/Event/EventData/Data[@Name=\"ProcessName\"]" + } + ] + } + ] +} diff --git a/crates/cmtraceopen-parser/tests/fixtures/eventmap/shell-core-9701.json b/crates/cmtraceopen-parser/tests/fixtures/eventmap/shell-core-9701.json new file mode 100644 index 000000000..ec895ed90 --- /dev/null +++ b/crates/cmtraceopen-parser/tests/fixtures/eventmap/shell-core-9701.json @@ -0,0 +1,19 @@ +{ + "Author": "Troy Larson", + "Description": "RunOnceEx commands started", + "EventId": 9701, + "Channel": "Microsoft-Windows-Shell-Core/Operational", + "Provider": "Microsoft-Windows-Shell-Core", + "Maps": [ + { + "Property": "PayloadData1", + "PropertyValue": "%PayloadData1%", + "Values": [ + { + "Name": "PayloadData1", + "Value": "/Event/EventData/Data" + } + ] + } + ] +} diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index fafd291c2..4ef6c64dc 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -25,7 +25,13 @@ tauri-build = { version = "2", features = [] } default = ["full"] full = ["collector", "deployment", "dsregcmd", "esp-diagnostics", "event-log", "intune-diagnostics", "macos-diag", "sccm-diagnostics", "secureboot", "sysmon"] sysmon = ["dep:evtx"] -event-log = ["dep:evtx"] +event-log = [ + "dep:evtx", + "dep:quick-xml", + "dep:serde_norway", + "dep:rusqlite", + "dep:flate2", +] collector = [] deployment = [] dsregcmd = ["intune-diagnostics"] @@ -75,6 +81,12 @@ cab = { version = "=0.6.0", optional = true } # using only std::time -- so the whole graph (cab, cookie, plist, simplelog, # tauri-plugin-log, tauri-plugin-updater) is held at or above the patched release. time = { version = "0.3.47", default-features = false, features = ["parsing"] } +# Only the event-log modules use these, so they are optional and pulled in by that feature. +# rusqlite carries "bundled", which compiles SQLite from source; an unrelated build should not pay +# for it. +serde_norway = { version = "0.9.42", optional = true } +rusqlite = { version = "0.40.2", features = ["bundled"], optional = true } +flate2 = { version = "1.1.9", optional = true } [target.'cfg(unix)'.dependencies] libc = "0.2" @@ -124,6 +136,12 @@ tempfile = "3" plist = "1" tauri = { version = "2", features = ["test"] } +# Timing harness for the live event-log scan. Declared so `--all-targets` does not try to build it +# without the modules it measures. +[[example]] +name = "evtx_scan" +required-features = ["event-log"] + [[test]] name = "esp_diagnostics_sources" required-features = ["esp-diagnostics"] diff --git a/src-tauri/examples/evtx_scan.rs b/src-tauri/examples/evtx_scan.rs new file mode 100644 index 000000000..cc1b7e6b4 --- /dev/null +++ b/src-tauri/examples/evtx_scan.rs @@ -0,0 +1,157 @@ +//! Times a live scan so a performance claim can be checked instead of asserted. +//! +//! This exists because the epic's Phase 1 gate requires a reproducible scenario with recorded +//! numbers, and because every change in the live path so far has been argued from reading the code +//! rather than from measurement. Wall clock is printed here; peak working set is measured by the +//! caller, since a process cannot observe its own peak as reliably as the parent can. +//! +//! Windows only, because there is no Event Log service to scan anywhere else. On other platforms it +//! prints why it did nothing rather than reporting a zero that would look like a fast scan. +//! +//! ```text +//! cargo run --release --example evtx_scan --features event-log -- --days 7 +//! cargo run --release --example evtx_scan --features event-log -- --days 7 --channel Application +//! ``` + +fn main() { + let args: Vec = std::env::args().collect(); + let value_of = |flag: &str| -> Option { + args.iter() + .position(|a| a == flag) + .and_then(|i| args.get(i + 1)) + .cloned() + }; + + let days: u64 = value_of("--days").and_then(|v| v.parse().ok()).unwrap_or(7); + let only_channel = value_of("--channel"); + // Absent means no cap, which is the case the gate cares about: the cap is the thing being + // replaced, so measuring with it in place would measure the cap rather than the scan. + let max_events: Option = value_of("--max").and_then(|v| v.parse().ok()); + + run(days, only_channel, max_events); +} + +#[cfg(target_os = "windows")] +fn run(days: u64, only_channel: Option, max_events: Option) { + use app_lib::event_log::live; + use cmtraceopen_parser::event_query::{EventQueryFilter, TimeWindow}; + use cmtraceopen_parser::eventmap::MapRegistry; + use std::time::Instant; + + let enumerated = Instant::now(); + let channels = match live::enumerate_channels() { + Ok(channels) => channels, + Err(error) => { + eprintln!("enumerate_channels failed: {error}"); + std::process::exit(1); + } + }; + let channels: Vec = channels + .into_iter() + .map(|c| c.name) + .filter(|name| only_channel.as_ref().is_none_or(|only| only == name)) + .collect(); + let enumerate_ms = enumerated.elapsed().as_millis(); + + let Some(milliseconds) = days.checked_mul(24 * 60 * 60 * 1000) else { + eprintln!("--days is too large"); + std::process::exit(2); + }; + + let filter = EventQueryFilter { + time: Some(TimeWindow::Last { milliseconds }), + ..Default::default() + }; + + // No maps loaded. The map engine has its own benchmark; mixing it in here would make a change + // to either one move this number. + let maps = MapRegistry::new(); + + let mut total = 0usize; + let mut failed = 0usize; + let mut slowest = (String::new(), 0u128); + // Attributing memory rather than guessing at it. Every record carries the whole rendered XML it + // was built from, and that string is then serialized to the frontend and held there too, so if + // it dominates the record it dominates three copies rather than one. + let mut xml_bytes = 0usize; + let mut field_bytes = 0usize; + let mut widest_channel = (String::new(), 0usize); + // A channel read only partly is neither a success nor a failure, and counting it as + // either hides it. The scan's own truncations are the thing most likely to make a + // faster number look like an improvement. Gap entries and affected channels are counted + // separately: one channel can report several gaps at once, so a count of entries would + // overstate the number of channels that came back incomplete. + let mut gap_reports = 0usize; + let mut channels_with_gaps = 0usize; + + let started = Instant::now(); + for channel in &channels { + let at = Instant::now(); + match live::query_channel_filtered(channel, &filter, &maps, max_events) { + Ok(scan) => { + let records = scan.records; + gap_reports += scan.gaps.len(); + if !scan.gaps.is_empty() { + channels_with_gaps += 1; + } + total += records.len(); + if records.len() > widest_channel.1 { + widest_channel = (channel.clone(), records.len()); + } + for record in &records { + xml_bytes += record.raw_xml.len(); + field_bytes += record.message.len() + + record + .event_data + .iter() + .map(|f| f.name.len() + f.value.len()) + .sum::(); + } + } + // A channel that cannot be read is counted, not ignored. Treating it as zero events + // would report a faster scan of a smaller corpus as an improvement. + Err(_) => failed += 1, + } + let took = at.elapsed().as_millis(); + if took > slowest.1 { + slowest = (channel.clone(), took); + } + } + let elapsed = started.elapsed(); + + let per_event_us = if total > 0 { + elapsed.as_micros() as f64 / total as f64 + } else { + 0.0 + }; + + println!("days={days}"); + println!("channels_scanned={}", channels.len()); + println!("channels_failed={failed}"); + println!("channels_with_gaps={channels_with_gaps}"); + println!("gap_entries={gap_reports}"); + println!("events={total}"); + println!("enumerate_ms={enumerate_ms}"); + println!("scan_ms={}", elapsed.as_millis()); + println!("per_event_us={per_event_us:.2}"); + println!("slowest_channel={} ({}ms)", slowest.0, slowest.1); + println!( + "widest_channel={} ({} events)", + widest_channel.0, widest_channel.1 + ); + println!("raw_xml_mb={:.1}", xml_bytes as f64 / (1024.0 * 1024.0)); + println!( + "message_and_fields_mb={:.1}", + field_bytes as f64 / (1024.0 * 1024.0) + ); + if xml_bytes + field_bytes > 0 { + let share = xml_bytes as f64 / (xml_bytes + field_bytes) as f64 * 100.0; + println!("raw_xml_share_pct={share:.0}"); + } +} + +#[cfg(not(target_os = "windows"))] +fn run(_days: u64, _only_channel: Option, _max_events: Option) { + eprintln!("evtx_scan needs a Windows Event Log service; nothing to measure on this platform."); + std::process::exit(2); +} diff --git a/src-tauri/src/commands/elevation.rs b/src-tauri/src/commands/elevation.rs index 2c5ea901c..660327755 100644 --- a/src-tauri/src/commands/elevation.rs +++ b/src-tauri/src/commands/elevation.rs @@ -111,9 +111,8 @@ impl Serialize for ElevationCommandError { match self { Self::InvalidRequest { reason } => map.serialize_entry("reason", reason)?, Self::Relaunch { source } => map.serialize_entry("source", source)?, - Self::AlreadyInProgress - | Self::TicketUnavailable - | Self::StateDirectoryUnavailable => {} + Self::AlreadyInProgress | Self::TicketUnavailable | Self::StateDirectoryUnavailable => { + } } map.end() } @@ -368,10 +367,9 @@ mod tests { #[test] fn ticket_operations_run_on_the_blocking_pool() { let caller = thread::current().id(); - let worker = tauri::async_runtime::block_on(run_blocking_operation(|| { - thread::current().id() - })) - .expect("blocking worker completes"); + let worker = + tauri::async_runtime::block_on(run_blocking_operation(|| thread::current().id())) + .expect("blocking worker completes"); assert_ne!(worker, caller, "ticket I/O must leave the calling thread"); } @@ -422,10 +420,7 @@ mod tests { .write(true) .open(&path) .expect("open ticket") - .set_times( - FileTimes::new() - .set_modified(SystemTime::now() + Duration::from_secs(60)), - ) + .set_times(FileTimes::new().set_modified(SystemTime::now() + Duration::from_secs(60))) .expect("set future mtime"); let abandoned = ticket_for( @@ -561,7 +556,8 @@ mod tests { let json = serde_json::to_value(&error).expect("serialize"); assert_eq!( - json["message"], expected, + json["message"], + expected, "{} lost its message", error.kind() ); diff --git a/src-tauri/src/commands/jamf.rs b/src-tauri/src/commands/jamf.rs index 1f7d18b3c..5d1ef03f2 100644 --- a/src-tauri/src/commands/jamf.rs +++ b/src-tauri/src/commands/jamf.rs @@ -2,8 +2,8 @@ use std::path::PathBuf; use crate::error::AppError; use crate::jamf::models::{ - JamfConnectEvent, JamfEnvironment, JamfLogScanResult, JamfPolicyLogResult, - JamfProfilesResult, JamfSelfServiceEvent, + JamfConnectEvent, JamfEnvironment, JamfLogScanResult, JamfPolicyLogResult, JamfProfilesResult, + JamfSelfServiceEvent, }; use crate::jamf::paths; use crate::macos_diag::environment::scan_log_directory; diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 0c69b0d47..e470644be 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -23,9 +23,9 @@ pub mod intune; pub mod intune_bundle; #[cfg(feature = "intune-diagnostics")] pub mod intune_diagnostics; -pub mod known_sources; #[cfg(feature = "macos-diag")] pub mod jamf; +pub mod known_sources; #[cfg(feature = "macos-diag")] pub mod macos_diag; pub mod markers; diff --git a/src-tauri/src/elevation/mod.rs b/src-tauri/src/elevation/mod.rs index eeb06dc09..a21d9ee96 100644 --- a/src-tauri/src/elevation/mod.rs +++ b/src-tauri/src/elevation/mod.rs @@ -507,9 +507,10 @@ mod tests { #[test] fn a_camel_case_known_source_request_deserializes() { - let target: RestoreTarget = - serde_json::from_value(serde_json::json!({ "kind": "knownSource", "sourceId": "ccm-logs" })) - .expect("camelCase is the wire contract"); + let target: RestoreTarget = serde_json::from_value( + serde_json::json!({ "kind": "knownSource", "sourceId": "ccm-logs" }), + ) + .expect("camelCase is the wire contract"); assert_eq!( target, @@ -522,8 +523,9 @@ mod tests { #[test] fn the_snake_case_known_source_form_is_rejected_rather_than_tolerated() { // Accepting both would let the contract drift back without a test failing. - let result: Result = - serde_json::from_value(serde_json::json!({ "kind": "knownSource", "source_id": "ccm-logs" })); + let result: Result = serde_json::from_value( + serde_json::json!({ "kind": "knownSource", "source_id": "ccm-logs" }), + ); assert!(result.is_err(), "snake_case must not be accepted"); } diff --git a/src-tauri/src/error.rs b/src-tauri/src/error.rs index 430256b3f..d0bf0707f 100644 --- a/src-tauri/src/error.rs +++ b/src-tauri/src/error.rs @@ -266,7 +266,10 @@ mod tests { #[test] fn access_denied_without_a_path_serializes_a_null_path() { - let value = payload(AppError::access_denied(SourceOperation::WorkspaceAction, None)); + let value = payload(AppError::access_denied( + SourceOperation::WorkspaceAction, + None, + )); assert_eq!(value["kind"], "accessDenied"); assert!(value["path"].is_null()); @@ -348,10 +351,7 @@ mod tests { ] { let message = AppError::access_denied(operation, None).to_string(); for os in ["Windows", "macOS", "Linux"] { - assert!( - !message.contains(os), - "{operation:?} names {os}: {message}" - ); + assert!(!message.contains(os), "{operation:?} names {os}: {message}"); } } } diff --git a/src-tauri/src/event_log/commands.rs b/src-tauri/src/event_log/commands.rs index 6e797c330..690a4ac02 100644 --- a/src-tauri/src/event_log/commands.rs +++ b/src-tauri/src/event_log/commands.rs @@ -6,6 +6,7 @@ use tauri::Emitter; use super::models::{EvtxChannelInfo, EvtxParseResult}; use super::parser; +use crate::state::app_state::AppState; #[cfg(target_os = "windows")] #[derive(Clone, Serialize)] @@ -15,9 +16,35 @@ struct EvtxQueryProgress { fetched: usize, } +/// A batch of records on its way to the frontend before the query has finished. +/// +/// One channel can be most of a scan: Security measured 286,401 of 404,769 events and 191.8 seconds +/// of 267, so a reply that waits for the channel to finish leaves an operator watching an empty +/// list for three minutes. Batches are emitted as they are read instead. +/// +/// `sequence` numbers the batches for one channel from zero. The receiver uses it to notice a batch +/// it never got: an event channel offers no delivery guarantee, and events that quietly failed to +/// arrive would look exactly like events that do not exist. +#[cfg(target_os = "windows")] +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct EvtxRecordBatch { + channel: String, + sequence: usize, + records: Vec, +} + #[tauri::command] -pub async fn evtx_parse_files(paths: Vec) -> Result { - tokio::task::spawn_blocking(move || parser::parse_evtx_files(&paths)) +pub async fn evtx_parse_files( + paths: Vec, + state: tauri::State<'_, AppState>, +) -> Result { + // Handles are cloned out before the blocking work starts. Parsing can run for a long time over + // a hundred thousand records, and holding a state lock across it would stall every other + // command. + let maps = state.event_maps.clone(); + let providers = state.provider_store.clone(); + tokio::task::spawn_blocking(move || parser::parse_evtx_files(&paths, &maps, &providers)) .await .map_err(|e| format!("Task join error: {}", e))? } @@ -40,35 +67,111 @@ pub async fn evtx_enumerate_channels() -> Result, String> { pub async fn evtx_query_channels( channels: Vec, max_events: Option, + filter: Option, app: AppHandle, + state: tauri::State<'_, AppState>, ) -> Result { #[cfg(target_os = "windows")] { + // Cloned before the blocking work, so a long channel scan never runs with a state lock + // held. Channels are then read under one shared guard rather than locking per record. + let registry = state.event_maps.clone(); tokio::task::spawn_blocking(move || { + use rayon::prelude::*; + + let maps = registry + .read() + .map_err(|_| "map registry lock was poisoned".to_string())?; + + // Absent means unfiltered, which keeps the query as "*" and preserves prior behaviour + // for callers that have not adopted server-side filtering yet. + let query_filter = filter.unwrap_or_default(); + + // Channels are queried concurrently. Each one is an independent conversation with the + // Event Log service that spends nearly all its time waiting on RPC, so serializing + // them leaves the machine idle. Results are collected per channel and ordered + // afterwards, so concurrency cannot affect the output. + let per_channel: Vec<(String, Result)> = channels + .par_iter() + .map(|channel| { + let app_ref = &app; + let ch_name = channel.clone(); + let batch_channel = channel.clone(); + let mut sequence = 0usize; + let outcome = super::live::query_channel_streamed( + channel, + &query_filter, + &maps, + max_events, + |fetched, _| { + let _ = app_ref.emit( + "evtx-query-progress", + EvtxQueryProgress { + channel: ch_name.clone(), + fetched, + }, + ); + }, + |batch| { + if batch.is_empty() { + return; + } + // Taken from the batch rather than copied out of it. Leaving the records + // behind would send them and also accumulate them, which is the memory + // this exists to avoid. + let records = std::mem::take(batch); + let emitted = app_ref.emit( + "evtx-record-batch", + EvtxRecordBatch { + channel: batch_channel.clone(), + sequence, + records, + }, + ); + sequence += 1; + if let Err(error) = emitted { + // The records are gone: they were moved into the payload. Say so + // rather than letting the count quietly disagree with the view. + log::warn!( + "event=evtx_batch_emit_failed channel=\"{batch_channel}\" \ + sequence={sequence} error=\"{error}\"" + ); + } + }, + ); + (channel.clone(), outcome) + }) + .collect(); + let mut all_records = Vec::new(); let mut channel_infos = Vec::new(); let mut parse_errors = 0u32; let mut error_messages = Vec::new(); + // Emitted in batches rather than carried in this reply. Reported so the receiver can + // check what it actually assembled against what was sent, instead of assuming an event + // channel delivered everything. + let mut streamed = 0usize; - for channel in &channels { - let app_ref = &app; - let ch_name = channel.clone(); - match super::live::query_channel_with_progress(channel, max_events, |fetched, _| { - let _ = app_ref.emit( - "evtx-query-progress", - EvtxQueryProgress { - channel: ch_name.clone(), - fetched, - }, - ); - }) { - Ok(records) => { + for (channel, outcome) in per_channel { + match outcome { + Ok(scan) => { + // `delivered`, not `records.len()`: the records were emitted in batches and + // taken out of the vector on the way. Counting what is left would report a + // fully read channel as empty. channel_infos.push(super::models::EvtxChannelInfo { name: channel.clone(), - event_count: records.len() as u64, + event_count: scan.delivered as u64, source_type: super::models::ChannelSourceType::Live, }); - all_records.extend(records); + streamed += scan.delivered; + // A channel can be read partly. Those gaps travel with the records so a + // truncated channel is not presented as a complete one, which is what + // happened while the query returned records alone. + if !scan.gaps.is_empty() { + parse_errors += scan.gaps.len() as u32; + error_messages.extend(scan.gaps); + } + all_records.extend(scan.records); } Err(e) => { log::warn!( @@ -77,7 +180,8 @@ pub async fn evtx_query_channels( e ); error_messages.push(format!("{}: {}", channel, e)); - // Still include channel in results with 0 events so frontend knows it was attempted + // A failed channel is still reported with 0 events so the gap stays visible + // rather than looking like a channel that simply had nothing in it. channel_infos.push(super::models::EvtxChannelInfo { name: channel.clone(), event_count: 0, @@ -88,12 +192,11 @@ pub async fn evtx_query_channels( } } + // `all_records` holds only what the batch hook left behind, which is nothing while the + // records are being streamed. `total_records` counts what was sent, so the receiver has + // something to check its own tally against. all_records.sort_by_key(|r| r.timestamp_epoch); - for (i, record) in all_records.iter_mut().enumerate() { - record.id = i as u64; - } - - let total_records = all_records.len() as u64; + let total_records = (streamed + all_records.len()) as u64; Ok(EvtxParseResult { records: all_records, @@ -108,7 +211,7 @@ pub async fn evtx_query_channels( } #[cfg(not(target_os = "windows"))] { - let _ = (channels, max_events, app); + let _ = (channels, max_events, filter, app, state); Ok(EvtxParseResult { records: Vec::new(), channels: Vec::new(), @@ -118,3 +221,128 @@ pub async fn evtx_query_channels( }) } } + +/// Writes `records` to `destination` in `format`. +/// +/// The records travel from the frontend rather than being re-queried, so what is exported is +/// exactly what the operator was looking at, including any client-side filtering they had applied. +/// Re-querying would risk exporting a different set than the one on screen. +#[tauri::command] +pub async fn evtx_export_records( + records: Vec, + format: super::export::ExportFormat, + destination: String, +) -> Result { + let record_count = records.len(); + // Rendered on a blocking thread, like every other heavy command here. The XML format + // concatenates raw_xml for up to a hundred thousand records into one String, which can occupy + // a runtime worker for seconds and stall unrelated IPC. + let rendered = + tokio::task::spawn_blocking(move || super::export::export_records(&records, format)) + .await + .map_err(|error| format!("export task failed: {error}"))??; + let byte_count = rendered.len() as u64; + tokio::fs::write(&destination, rendered) + .await + .map_err(|error| format!("cannot write {destination}: {error}"))?; + log::info!( + "event=evtx_export destination=\"{destination}\" records={record_count} bytes={byte_count}" + ); + Ok(byte_count) +} + +/// Loads EvtxECmd `.map` files from `directory` into the application's registry. +/// +/// Returns what loaded, what was superseded, and what failed, so an operator can see why an event +/// type is not being mapped rather than being left guessing. +#[tauri::command] +pub async fn evtx_load_event_maps( + directory: String, + state: tauri::State<'_, AppState>, +) -> Result { + let path = std::path::PathBuf::from(&directory); + let maps = state.event_maps.clone(); + tokio::task::spawn_blocking(move || { + // Read from disk first, then swap. Holding the write lock across the read would block + // every in-flight parse for as long as the directory takes to load. + let (registry, outcome) = super::maps::load_maps_from_dir(&path)?; + *maps + .write() + .map_err(|_| "map registry lock was poisoned".to_string())? = registry; + Ok(outcome) + }) + .await + .map_err(|error| format!("map load task failed: {error}"))? +} + +/// Number of maps currently in effect. +#[tauri::command] +pub async fn evtx_loaded_map_count(state: tauri::State<'_, AppState>) -> Result { + Ok(state + .event_maps + .read() + .map_err(|_| "map registry lock was poisoned".to_string())? + .len() as u64) +} + +/// Registers every provider database in `directory` for description rendering. +/// +/// Returns a summary per database so an operator can see what coverage was actually gained rather +/// than assuming a directory full of files worked. +#[tauri::command] +pub async fn evtx_load_provider_databases( + directory: String, + state: tauri::State<'_, AppState>, +) -> Result, String> { + let path = std::path::PathBuf::from(&directory); + let providers = state.provider_store.clone(); + tokio::task::spawn_blocking( + move || -> Result, String> { + // Scanned into a fresh store first, then swapped, so the write lock is held for the + // assignment rather than for the whole directory walk. Any parse in flight would + // otherwise block on its read guard for as long as opening every database takes. Same + // rule the map registry follows above. + let mut loaded = super::provider_db::ProviderStore::default(); + let info = loaded.load_directory(&path)?; + *providers + .write() + .map_err(|_| "provider store lock was poisoned".to_string())? = loaded; + Ok(info) + }, + ) + .await + .map_err(|error| format!("provider database load task failed: {error}"))? +} + +/// Provider databases currently registered. +#[tauri::command] +pub async fn evtx_provider_databases( + state: tauri::State<'_, AppState>, +) -> Result, String> { + Ok(state + .provider_store + .read() + .map_err(|_| "provider store lock was poisoned".to_string())? + .registered()) +} + +/// Merges already-loaded log entries and event records into one chronological timeline. +/// +/// Both sides arrive from the frontend rather than being re-read, so the timeline covers exactly +/// what the operator has open. Re-reading would risk building a timeline from a different set than +/// the one on screen. +#[tauri::command] +pub async fn evtx_build_unified_timeline( + entries: Vec, + records: Vec, +) -> Result { + let timeline = tokio::task::spawn_blocking(move || super::timeline::build(&entries, &records)) + .await + .map_err(|error| format!("timeline build task failed: {error}"))?; + log::info!( + "event=unified_timeline items={} unplaced={}", + timeline.items.len(), + timeline.unplaced.len() + ); + Ok(timeline) +} diff --git a/src-tauri/src/event_log/event_node.rs b/src-tauri/src/event_log/event_node.rs new file mode 100644 index 000000000..869382a21 --- /dev/null +++ b/src-tauri/src/event_log/event_node.rs @@ -0,0 +1,504 @@ +//! Converting rendered event XML into the parser crate's [`EventNode`] tree. +//! +//! `cmtraceopen-parser` is wasm32-compatible and carries no XML reader, so the map engine takes an +//! already-parsed tree. This is the host-side adapter that produces one from the XML that +//! `EvtRender` and the `evtx` crate both emit. +//! +//! Namespace prefixes are stripped. Event XML declares a default namespace and providers +//! occasionally emit prefixed elements, but map path expressions are written without prefixes +//! (`/Event/EventData/Data`), so keeping them would make every such map silently match nothing. + +use super::models::EvtxField; +use super::sanitize_control_chars; +use cmtraceopen_parser::eventmap::EventNode; +use quick_xml::events::Event as XmlEvent; +use quick_xml::{Reader, XmlVersion}; + +/// Parses rendered event XML into a node tree rooted at `Event`. +/// +/// Returns `Err` only when the document is not well formed. A document that is well formed but +/// unexpected in shape yields whatever tree it describes, so a provider emitting something unusual +/// degrades to unmapped columns rather than to a failed query. +pub fn parse_event_xml(xml: &str) -> Result { + let mut reader = Reader::from_str(xml); + // Text is not trimmed globally. Trimming would strip meaningful spaces from field values, and + // because entity references arrive as separate events, "a & b" would be reassembled from + // individually trimmed fragments as "a&b". Whitespace-only text on a container element is + // dropped when the element closes instead, which removes pretty-printing without touching + // real content. + reader.config_mut().trim_text(false); + + let mut stack: Vec = Vec::new(); + let mut root: Option = None; + let mut buffer = Vec::new(); + + loop { + match reader.read_event_into(&mut buffer) { + Ok(XmlEvent::Start(start)) => stack.push(element_from(&start)?), + Ok(XmlEvent::Empty(empty)) => { + let node = element_from(&empty)?; + close(&mut stack, &mut root, node); + } + Ok(XmlEvent::Text(text)) => { + let value = text + .xml10_content() + .map_err(|error| format!("undecodable text: {error}"))? + .into_owned(); + push_text(&mut stack, &value); + } + Ok(XmlEvent::GeneralRef(reference)) => { + // Entity references are their own event in quick-xml 0.41. Ignoring them would + // silently drop every '&', '<' and '>' from event data, which is common in command + // lines and file paths. + let name = reference + .decode() + .map_err(|error| format!("undecodable entity reference: {error}"))? + .into_owned(); + let raw = format!("&{name};"); + let resolved = quick_xml::escape::unescape(&raw) + .map(|value| value.into_owned()) + .unwrap_or(raw); + push_text(&mut stack, &resolved); + } + Ok(XmlEvent::CData(data)) => { + let value = String::from_utf8_lossy(&data).into_owned(); + push_text(&mut stack, &value); + } + Ok(XmlEvent::End(_)) => { + let Some(node) = stack.pop() else { continue }; + close(&mut stack, &mut root, node); + } + Ok(XmlEvent::Eof) => break, + Ok(_) => {} + Err(error) => return Err(format!("malformed event XML: {error}")), + } + buffer.clear(); + } + + root.ok_or_else(|| "event XML contained no elements".to_string()) +} + +fn push_text(stack: &mut [EventNode], value: &str) { + if let Some(current) = stack.last_mut() { + match current.text.as_mut() { + Some(existing) => existing.push_str(value), + None => current.text = Some(value.to_string()), + } + } +} + +fn close(stack: &mut [EventNode], root: &mut Option, mut node: EventNode) { + // Pretty-printed XML puts newlines and indentation inside container elements. That is layout, + // not content, so it is dropped once we know the element has children. + if !node.children.is_empty() + && node + .text + .as_deref() + .is_some_and(|text| text.trim().is_empty()) + { + node.text = None; + } + + match stack.last_mut() { + Some(parent) => parent.children.push(node), + // The outermost element is the root. Later siblings at depth zero are ignored rather than + // replacing it, so a stray trailing element cannot discard the event. + None => { + if root.is_none() { + *root = Some(node); + } + } + } +} + +fn element_from(start: &quick_xml::events::BytesStart<'_>) -> Result { + let mut node = EventNode::new(local_name(start.name().as_ref())); + for attribute in start.attributes() { + let attribute = attribute.map_err(|error| format!("malformed attribute: {error}"))?; + let name = local_name(attribute.key.as_ref()); + let value = attribute + .normalized_value(XmlVersion::Implicit1_0) + .map_err(|error| format!("undecodable attribute value: {error}"))? + .into_owned(); + node.attributes.push((name, value)); + } + Ok(node) +} + +fn local_name(raw: &[u8]) -> String { + let text = String::from_utf8_lossy(raw); + match text.rsplit_once(':') { + Some((_, local)) => local.to_string(), + None => text.into_owned(), + } +} + +/// System-block fields that every event carries, regardless of provider. +/// +/// Unlike map-derived columns, which only exist where someone has written a map, these are present +/// on every event, so they are extracted unconditionally. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SystemFields { + pub provider: Option, + pub event_id: Option, + pub level: Option, + pub channel: Option, + pub computer: Option, + pub time_created: Option, + pub event_record_id: Option, + pub task: Option, + pub opcode: Option, + pub process_id: Option, + pub thread_id: Option, + pub user_sid: Option, + pub keywords: Option, +} + +/// Reads the `System` block of a parsed event. +/// +/// Every field is optional because providers legitimately omit them: a task of zero is commonly +/// written as no element at all, and `Security` carries no `UserID` for events raised outside a +/// user context. An absent field stays `None` rather than defaulting to zero, which would claim +/// the provider said something it did not. +pub fn extract_system_fields(root: &EventNode) -> SystemFields { + let Some(system) = root.children.iter().find(|child| child.name == "System") else { + return SystemFields::default(); + }; + + let text_of = |name: &str| -> Option<&str> { + system + .children + .iter() + .find(|child| child.name == name) + .and_then(|child| child.text.as_deref()) + .map(str::trim) + .filter(|value| !value.is_empty()) + }; + let attribute_of = |element: &str, attribute: &str| -> Option { + system + .children + .iter() + .find(|child| child.name == element) + .and_then(|child| child.attribute(attribute)) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + }; + + SystemFields { + // Manifest providers write Name; classic sources write only EventSourceName, for example + // . Reading just Name left every classic + // event with provider "Unknown", which meant no map could match it and no description + // could be rendered for it. + provider: attribute_of("Provider", "Name") + .or_else(|| attribute_of("Provider", "EventSourceName")), + // Classic providers write `1000`. The id is the + // element text in both shapes; the qualifier is separate and not part of the id. + event_id: text_of("EventID").and_then(|value| value.parse().ok()), + level: text_of("Level").and_then(|value| value.parse().ok()), + channel: text_of("Channel").map(str::to_string), + computer: text_of("Computer").map(str::to_string), + time_created: attribute_of("TimeCreated", "SystemTime"), + event_record_id: text_of("EventRecordID").and_then(|value| value.parse().ok()), + task: text_of("Task").and_then(|value| value.parse().ok()), + opcode: text_of("Opcode").and_then(|value| value.parse().ok()), + process_id: attribute_of("Execution", "ProcessID").and_then(|v| v.parse().ok()), + thread_id: attribute_of("Execution", "ThreadID").and_then(|v| v.parse().ok()), + user_sid: attribute_of("Security", "UserID"), + keywords: text_of("Keywords").map(str::to_string), + } +} + +/// What an event's data section yielded, in the two shapes that are needed. +/// +/// They differ, and conflating them corrupts messages. The display list omits fields the provider +/// left empty, because a column of blanks is noise. The insertion list keeps them, because the +/// message template addresses fields by position and a gap shifts every later reference. +pub struct EventFields { + pub fields: Vec, + pub insertions: Vec, +} + +/// Extract event fields as name-value pairs. +/// +/// Both `EventData` and `UserData` are read. Manifest providers use the former and classic or +/// trace-backed providers the latter, and skipping `UserData` would leave those events with no +/// fields at all. +/// +/// A `Data` element with no `Name` attribute is numbered by its position, matching how the event +/// message template refers to it. Values are sanitized to strip control characters that would +/// render as unexpected glyphs. +pub fn extract_event_data(root: &EventNode) -> EventFields { + let mut fields = Vec::new(); + let mut insertions = Vec::new(); + let mut unnamed = 0usize; + + let containers = root + .children + .iter() + .filter(|child| child.name == "EventData" || child.name == "UserData"); + + // A child that carries text is a field. A child that carries only elements is a wrapper, which + // is how UserData nests its fields under a provider-named element, so it is descended through. + // + // Deciding per child rather than per container matters: an EventData holding only has + // no at all, and treating the whole container as wrappers would descend into , + // find no children, and drop the only value the event carried. + let push_field = |child: &EventNode, + fields: &mut Vec, + insertions: &mut Vec, + unnamed: &mut usize| { + let value = sanitize_control_chars(child.text.as_deref().unwrap_or_default()); + // Recorded even when empty. The provider's message template addresses fields by position, + // so skipping one here shifts every later %N and renders the description with the wrong + // values substituted into it, which reads as fact. + insertions.push(value.clone()); + + // Counted before the emptiness check, for the same reason. The label is what an operator + // uses to match a field against the provider's template, so skipping the count for a blank + // slot would label the second field Data1 while the template calls it %2. + let position = if child.attribute("Name").is_none() && child.name == "Data" { + *unnamed += 1; + Some(*unnamed) + } else { + None + }; + + if value.is_empty() { + return; + } + let name = match (child.attribute("Name"), position) { + (Some(name), _) => name.to_string(), + (None, Some(position)) => format!("Data{position}"), + (None, None) => child.name.clone(), + }; + fields.push(EvtxField { name, value }); + }; + + for container in containers { + for child in &container.children { + if child.text.is_some() || child.children.is_empty() { + push_field(child, &mut fields, &mut insertions, &mut unnamed); + } else { + for grandchild in &child.children { + push_field(grandchild, &mut fields, &mut insertions, &mut unnamed); + } + } + } + } + + EventFields { fields, insertions } +} + +#[cfg(test)] +mod tests { + use super::*; + use cmtraceopen_parser::eventmap::ValuePath; + + /// Shape emitted by EvtRender for a real Security 4624, trimmed to the interesting parts. + const RENDERED: &str = r#" + + + 4624 + 0 + + + Security + TESTHOST-01 + + + adam + 10 + + +"#; + + fn resolve(xml: &str, path: &str) -> Option { + let root = parse_event_xml(xml).expect("parses"); + ValuePath::parse(path) + .expect("path parses") + .evaluate(&root) + .map(|value| value.into_owned()) + } + + #[test] + fn the_root_is_the_event_element() { + let root = parse_event_xml(RENDERED).expect("parses"); + assert_eq!(root.name, "Event"); + assert_eq!(root.children.len(), 2); + } + + #[test] + fn named_event_data_resolves_through_a_map_path() { + assert_eq!( + resolve(RENDERED, r#"/Event/EventData/Data[@Name="LogonType"]"#).as_deref(), + Some("10") + ); + } + + #[test] + fn self_closing_elements_keep_their_attributes() { + assert_eq!( + resolve(RENDERED, "/Event/System/Provider/@Name").as_deref(), + Some("Microsoft-Windows-Security-Auditing") + ); + assert_eq!( + resolve(RENDERED, "/Event/System/Correlation/@ActivityID").as_deref(), + Some("{2f8b0c1e-0000-0000-0000-000000000000}") + ); + } + + #[test] + fn element_text_resolves() { + assert_eq!( + resolve(RENDERED, "/Event/System/Computer").as_deref(), + Some("TESTHOST-01") + ); + } + + #[test] + fn a_namespace_prefix_is_stripped_so_unprefixed_map_paths_still_match() { + let prefixed = r#"v"#; + assert_eq!( + resolve(prefixed, r#"/Event/EventData/Data[@Name="X"]"#).as_deref(), + Some("v") + ); + } + + #[test] + fn repeated_unnamed_data_is_preserved_for_the_engine_to_join() { + let xml = + "abc"; + assert_eq!( + resolve(xml, "/Event/EventData/Data").as_deref(), + Some("a, b, c") + ); + } + + #[test] + fn an_empty_element_is_present_but_has_no_text() { + let root = parse_event_xml(RENDERED).expect("parses"); + let event_data = root + .children + .iter() + .find(|child| child.name == "EventData") + .expect("EventData present"); + let empty = event_data + .children + .iter() + .find(|child| child.attribute("Name") == Some("Empty")) + .expect("empty Data present"); + assert_eq!(empty.text, None); + } + + #[test] + fn escaped_entities_are_decoded() { + let xml = + r#"a & b <c>"#; + assert_eq!( + resolve(xml, r#"/Event/EventData/Data[@Name="Cmd"]"#).as_deref(), + Some("a & b ") + ); + } + + #[test] + fn cdata_content_is_captured() { + let xml = + "]]>"; + assert_eq!( + resolve(xml, r#"/Event/EventData/Data[@Name="X"]"#).as_deref(), + Some("raw ") + ); + } + + #[test] + fn system_fields_are_read_from_a_full_event() { + let xml = r#" + 13312110x8020000000000000 + + + "#; + let fields = extract_system_fields(&parse_event_xml(xml).expect("parses")); + assert_eq!(fields.task, Some(13312)); + assert_eq!(fields.opcode, Some(11)); + assert_eq!(fields.process_id, Some(1234)); + assert_eq!(fields.thread_id, Some(5678)); + assert_eq!(fields.user_sid.as_deref(), Some("S-1-5-18")); + assert_eq!(fields.keywords.as_deref(), Some("0x8020000000000000")); + } + + #[test] + fn an_omitted_field_stays_none_rather_than_defaulting_to_zero() { + // Claiming task 0 when the provider wrote no Task element would be inventing evidence. + let xml = "1"; + let fields = extract_system_fields(&parse_event_xml(xml).expect("parses")); + assert_eq!( + fields, + SystemFields { + event_id: Some(1), + ..SystemFields::default() + } + ); + } + + #[test] + fn an_empty_security_element_yields_no_sid() { + let xml = r#""#; + let fields = extract_system_fields(&parse_event_xml(xml).expect("parses")); + assert_eq!(fields.user_sid, None); + } + + #[test] + fn a_non_numeric_task_is_ignored_rather_than_failing_the_record() { + let xml = "not-a-number"; + let fields = extract_system_fields(&parse_event_xml(xml).expect("parses")); + assert_eq!(fields.task, None); + } + + #[test] + fn an_event_without_a_system_block_yields_defaults() { + let xml = "x"; + assert_eq!( + extract_system_fields(&parse_event_xml(xml).expect("parses")), + SystemFields::default() + ); + } + + #[test] + fn malformed_xml_is_an_error_rather_than_a_partial_tree() { + // Asserted separately. Joined with `||` the test passed on either input, so it never + // established that an unclosed element is rejected; if that started being accepted the + // second disjunct kept it green. + assert!( + parse_event_xml("").is_err(), + "an unclosed element must not yield a partial tree" + ); + assert!(parse_event_xml(" with no Name is what classic sources emit. Reading + // only Name left these as "Unknown", so no map matched and no description rendered. + let xml = r#" + + 1000 + "#; + let fields = extract_system_fields(&parse_event_xml(xml).expect("parses")); + assert_eq!(fields.provider.as_deref(), Some("Application Error")); + } + + #[test] + fn a_manifest_provider_still_wins_on_name() { + // Some events carry both; Name is the modern identity and must take precedence. + let xml = r#" + + "#; + let fields = extract_system_fields(&parse_event_xml(xml).expect("parses")); + assert_eq!( + fields.provider.as_deref(), + Some("Microsoft-Windows-Kernel-General") + ); + } +} diff --git a/src-tauri/src/event_log/export.rs b/src-tauri/src/event_log/export.rs new file mode 100644 index 000000000..401bc4d11 --- /dev/null +++ b/src-tauri/src/event_log/export.rs @@ -0,0 +1,515 @@ +//! Exporting event records to text formats. +//! +//! FullEventLogView offers nine export formats and we offered none, which made every analysis +//! dead-end in the app. This covers the three that carry the data losslessly enough to be worth +//! having: CSV for spreadsheets, JSON for tooling, and raw event XML for anything that wants the +//! provider's own representation. +//! +//! Formatting is deliberately separate from writing files, so the rules below are unit-testable +//! without touching the filesystem. + +use serde::{Deserialize, Serialize}; + +use super::models::EvtxRecord; + +/// A supported export format. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ExportFormat { + /// Comma-separated, one row per event, with a header line. + Csv, + /// Tab-separated, one row per event, with a header line. + Tsv, + /// A JSON array of the full records. + Json, + /// The provider's own event XML, concatenated under a single root. + Xml, +} + +impl ExportFormat { + /// The conventional file extension. + pub fn extension(&self) -> &'static str { + match self { + Self::Csv => "csv", + Self::Tsv => "tsv", + Self::Json => "json", + Self::Xml => "xml", + } + } + + fn delimiter(&self) -> char { + match self { + Self::Tsv => '\t', + _ => ',', + } + } +} + +/// Columns every delimited export carries, in order. +/// +/// `User SID` is here because it is a primary pivot for event triage; leaving it out meant an +/// analyst who exported the grid got fewer columns than the grid had shown them. +const COLUMNS: [&str; 14] = [ + "Event Time", + "Record ID", + "Event ID", + "Level", + "Provider", + "Channel", + "Computer", + "Task", + "Opcode", + "Process ID", + "Thread ID", + "User SID", + "Keywords", + "Description", +]; + +/// Map-derived column names present across `records`, in first-seen order. +/// +/// Appended after the fixed columns so a delimited export carries the same map values the grid +/// renders. Discovered from the records rather than declared, because which properties exist +/// depends on which maps matched. +fn mapped_columns(records: &[EvtxRecord]) -> Vec { + let mut names: Vec = Vec::new(); + for record in records { + for column in &record.mapped { + if !names.iter().any(|existing| existing == &column.property) { + names.push(column.property.clone()); + } + } + } + names +} + +/// Neutralizes a value that a spreadsheet would otherwise execute as a formula. +/// +/// Excel and LibreOffice treat a leading `=`, `+`, `-` or `@` as the start of a formula. Event +/// descriptions and command lines routinely begin with those characters, and an attacker who can +/// influence event content can otherwise get code to run when an analyst opens the export. A +/// leading apostrophe forces the cell to be read as text. +fn neutralize_formula(value: &str) -> String { + match value.chars().next() { + Some('=') | Some('+') | Some('-') | Some('@') => format!("'{value}"), + // Tab and carriage return are also treated as formula leads by some spreadsheet readers. + Some('\t') | Some('\r') => format!("'{value}"), + _ => value.to_string(), + } +} + +/// Quotes a field for delimiter-separated output. +/// +/// Always quoting would be simpler, but unquoted values keep exports diffable, so quoting is +/// applied only where the value would otherwise break the row. +fn escape_delimited(value: &str, delimiter: char) -> String { + let value = neutralize_formula(value); + let needs_quotes = value.contains(delimiter) + || value.contains('"') + || value.contains('\n') + || value.contains('\r'); + if needs_quotes { + format!("\"{}\"", value.replace('"', "\"\"")) + } else { + value + } +} + +/// Removes a leading `` declaration. +/// +/// Returns the input unchanged when there is none, and leaves everything after the declaration +/// exactly as the source wrote it. +fn strip_xml_declaration(xml: &str) -> &str { + let trimmed = xml.trim_start(); + let Some(rest) = trimmed.strip_prefix("") { + Some(end) => rest[end + 2..].trim_start(), + // A declaration that never closes is not something to silently repair. + None => xml, + } +} + +fn optional(value: Option) -> String { + value.map(|v| v.to_string()).unwrap_or_default() +} + +fn row_of(record: &EvtxRecord, mapped: &[String]) -> Vec { + let mut row: Vec = vec![ + record.timestamp.clone(), + record.event_record_id.to_string(), + record.event_id.to_string(), + format!("{:?}", record.level), + record.provider.clone(), + record.channel.clone(), + record.computer.clone(), + optional(record.task), + optional(record.opcode), + optional(record.process_id), + optional(record.thread_id), + record.user_sid.clone().unwrap_or_default(), + record.keywords.clone().unwrap_or_default(), + record.message.clone(), + ]; + for property in mapped { + // An incomplete mapping renders empty here for the same reason it does in the grid: a + // half-substituted template would put a literal %3 into an exported cell. + let value = record + .mapped + .iter() + .find(|column| &column.property == property) + .filter(|column| column.complete) + .map(|column| column.text.clone()) + .unwrap_or_default(); + row.push(value); + } + row +} + +/// Renders records in `format`. +pub fn export_records(records: &[EvtxRecord], format: ExportFormat) -> Result { + match format { + ExportFormat::Csv | ExportFormat::Tsv => { + let delimiter = format.delimiter(); + let mapped = mapped_columns(records); + + // Written straight into the output. Collecting each row into a Vec and joining it + // allocated a vector and a fresh separator String per record, on an export that can + // run to a hundred thousand of them. + let mut out = String::new(); + let write_row = |out: &mut String, cells: &mut dyn Iterator| { + let mut first = true; + for cell in cells { + if !first { + out.push(delimiter); + } + first = false; + out.push_str(&escape_delimited(cell, delimiter)); + } + out.push('\n'); + }; + + write_row( + &mut out, + &mut COLUMNS + .iter() + .copied() + .chain(mapped.iter().map(String::as_str)), + ); + for record in records { + let row = row_of(record, &mapped); + write_row(&mut out, &mut row.iter().map(String::as_str)); + } + Ok(out) + } + ExportFormat::Json => { + serde_json::to_string_pretty(records).map_err(|error| error.to_string()) + } + ExportFormat::Xml => { + let mut out = String::from("\n\n"); + for record in records { + // The provider's own XML is passed through otherwise untouched: re-encoding it + // would risk changing what the source actually said, which matters when an export + // is evidence. Only the per-record declaration is removed, because the evtx reader + // prefixes every record with one and a declaration is legal only at the very start + // of a document. Concatenating them produced a file no XML parser would open. + out.push_str(strip_xml_declaration(record.raw_xml.trim())); + out.push('\n'); + } + out.push_str("\n"); + Ok(out) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::event_log::models::EvtxLevel; + + fn record(message: &str) -> EvtxRecord { + EvtxRecord { + id: 0, + event_record_id: 42, + timestamp: "2026-08-09 12:00:00".into(), + timestamp_epoch: 0, + provider: "ESENT".into(), + channel: "Application".into(), + event_id: 326, + level: EvtxLevel::Error, + computer: "TESTHOST-01".into(), + message: message.into(), + event_data: Vec::new(), + raw_xml: "".into(), + source_label: "Live".into(), + task: Some(13312), + opcode: None, + process_id: Some(1234), + thread_id: None, + user_sid: Some("S-1-5-18".into()), + keywords: Some("0x80".into()), + mapped: Vec::new(), + } + } + + fn csv_body(message: &str) -> String { + let out = export_records(&[record(message)], ExportFormat::Csv).expect("exports"); + out.lines().nth(1).expect("data row").to_string() + } + + #[test] + fn csv_starts_with_a_header_row() { + let out = export_records(&[], ExportFormat::Csv).expect("exports"); + assert!(out.starts_with("Event Time,Record ID,Event ID,Level,")); + } + + #[test] + fn a_value_containing_the_delimiter_is_quoted() { + assert!(csv_body("a, b").contains("\"a, b\"")); + } + + #[test] + fn embedded_quotes_are_doubled() { + assert!(csv_body("say \"hi\"").contains("\"say \"\"hi\"\"\"")); + } + + #[test] + fn a_newline_inside_a_value_is_quoted_rather_than_breaking_the_row() { + let out = export_records(&[record("line1\nline2")], ExportFormat::Csv).expect("exports"); + assert!(out.contains("\"line1\nline2\"")); + // Header plus one logical row; the embedded newline must stay inside the quoted field. + assert_eq!(out.matches("2026-08-09 12:00:00").count(), 1); + } + + #[test] + fn a_leading_equals_cannot_execute_as_a_spreadsheet_formula() { + // Event content is attacker-influenceable, and an analyst opening the export in Excel + // would otherwise run it. + let body = csv_body("=cmd|'/c calc'!A1"); + assert!(body.contains("'=cmd"), "{body}"); + assert!(!body.contains(",=cmd"), "{body}"); + } + + #[test] + fn the_other_formula_leads_are_neutralized_too() { + for lead in ['+', '-', '@'] { + let body = csv_body(&format!("{lead}danger")); + assert!(body.contains(&format!("'{lead}danger")), "{body}"); + } + } + + #[test] + fn an_ordinary_value_is_not_quoted_so_exports_stay_diffable() { + let body = csv_body("plain text"); + assert!(body.contains(",plain text"), "{body}"); + } + + #[test] + fn an_absent_optional_renders_empty_rather_than_zero() { + // Opcode and thread id are None on the fixture; claiming 0 would invent provider data. + let body = csv_body("x"); + let fields: Vec<&str> = body.split(',').collect(); + assert_eq!(fields[7], "13312", "task present"); + assert_eq!(fields[8], "", "opcode absent"); + assert_eq!(fields[10], "", "thread id absent"); + } + + #[test] + fn the_user_sid_reaches_the_export() { + // A primary pivot for triage. Leaving it out gave an analyst fewer columns than the grid + // had shown them. + let out = export_records(&[record("x")], ExportFormat::Csv).expect("exports"); + assert!(out.lines().next().expect("header").contains("User SID")); + assert!(out.lines().nth(1).expect("row").contains("S-1-5-18")); + } + + #[test] + fn map_derived_columns_are_appended_after_the_fixed_ones() { + let mut mapped = record("x"); + mapped.mapped = vec![crate::event_log::maps::MappedColumn { + property: "PayloadData1".into(), + text: "cmd.exe".into(), + complete: true, + }]; + + let out = export_records(&[mapped], ExportFormat::Csv).expect("exports"); + let header = out.lines().next().expect("header"); + assert!(header.ends_with("PayloadData1"), "{header}"); + assert!(out.lines().nth(1).expect("row").ends_with("cmd.exe")); + } + + #[test] + fn a_record_without_a_mapped_value_leaves_the_cell_empty() { + // The union of properties is the header, so a record the map did not match still has to + // line up with it. + let mut mapped = record("has one"); + mapped.mapped = vec![crate::event_log::maps::MappedColumn { + property: "PayloadData1".into(), + text: "cmd.exe".into(), + complete: true, + }]; + let plain = record("has none"); + + let out = export_records(&[mapped, plain], ExportFormat::Csv).expect("exports"); + let rows: Vec<&str> = out.lines().collect(); + let columns = |line: &str| line.split(',').count(); + assert_eq!(columns(rows[0]), columns(rows[1])); + assert_eq!(columns(rows[0]), columns(rows[2])); + assert!( + rows[2].ends_with(','), + "the unmatched cell is empty: {}", + rows[2] + ); + } + + #[test] + fn an_incomplete_mapping_exports_empty_rather_than_a_raw_template() { + let mut mapped = record("x"); + mapped.mapped = vec![crate::event_log::maps::MappedColumn { + property: "PayloadData1".into(), + text: "ran %3 as adam".into(), + complete: false, + }]; + + let out = export_records(&[mapped], ExportFormat::Csv).expect("exports"); + assert!(!out.contains("%3"), "{out}"); + } + + #[test] + fn tsv_uses_tabs_and_quotes_values_containing_them() { + let out = export_records(&[record("a\tb")], ExportFormat::Tsv).expect("exports"); + assert!(out.starts_with("Event Time\tRecord ID")); + assert!(out.contains("\"a\tb\"")); + } + + #[test] + fn json_carries_the_metadata_fields_with_the_wire_names_the_frontend_reads() { + // The TypeScript EvtxRecord declares these in camelCase. Nothing on either side compares + // the two, so a rename or a missed serde attribute would surface only as undefined in the + // detail pane. + let mut r = record("x"); + r.mapped = vec![crate::event_log::maps::MappedColumn { + property: "PayloadData1".into(), + text: "cmd.exe".into(), + complete: true, + }]; + let json: serde_json::Value = + serde_json::from_str(&export_records(&[r], ExportFormat::Json).expect("exports")) + .expect("valid JSON"); + let first = &json[0]; + + // Presence, not value: the fixture leaves some of these unset on purpose, and an absent + // optional legitimately serializes as null. What matters is that the key is there and + // spelled the way the frontend reads it. + for key in [ + "eventRecordId", + "timestampEpoch", + "sourceLabel", + "processId", + "threadId", + "userSid", + "eventData", + "rawXml", + "mapped", + ] { + assert!(first.get(key).is_some(), "{key} missing: {first}"); + } + assert_eq!(first["eventRecordId"], 42); + assert_eq!(first["processId"], 1234); + assert_eq!(first["userSid"], "S-1-5-18"); + assert_eq!(first["mapped"][0]["property"], "PayloadData1"); + for snake in [ + "event_record_id", + "timestamp_epoch", + "source_label", + "user_sid", + ] { + assert!(first.get(snake).is_none(), "{snake} leaked in snake_case"); + } + } + + #[test] + fn json_round_trips_back_into_records() { + let out = export_records(&[record("hello")], ExportFormat::Json).expect("exports"); + let restored: Vec = serde_json::from_str(&out).expect("valid JSON"); + assert_eq!(restored.len(), 1); + assert_eq!(restored[0].event_record_id, 42); + assert_eq!(restored[0].task, Some(13312)); + assert_eq!(restored[0].opcode, None); + } + + #[test] + fn xml_passes_the_provider_representation_through_untouched() { + let out = export_records(&[record("x")], ExportFormat::Xml).expect("exports"); + assert!(out.starts_with("")); + assert!(out.contains("")); + assert!(out.contains("")); + assert!(out.trim_end().ends_with("")); + } + + #[test] + fn per_record_xml_declarations_are_stripped_from_the_concatenation() { + // The evtx reader prefixes every record with a declaration, and a declaration is legal + // only at the very start of a document. Concatenating them produced a file no XML parser + // would open. + let mut record = record("x"); + record.raw_xml = + "\n".into(); + let out = export_records(&[record.clone(), record], ExportFormat::Xml).expect("exports"); + + assert_eq!( + out.matches("")); + assert_eq!(out.matches("").count(), 2, "both records survive"); + } + + #[test] + fn a_record_without_a_declaration_is_untouched() { + let out = export_records(&[record("x")], ExportFormat::Xml).expect("exports"); + assert!(out.contains("")); + } + + #[test] + fn an_empty_export_is_still_well_formed() { + assert!(export_records(&[], ExportFormat::Json).expect("exports") == "[]"); + let xml = export_records(&[], ExportFormat::Xml).expect("exports"); + assert!(xml.contains("") && xml.contains("")); + } + + #[test] + fn a_delimited_export_works_from_a_payload_without_raw_xml() { + // The frontend omits rawXml for CSV and TSV because neither reads it and it dominates the + // IPC payload. Deserializing has to tolerate that, and the output must be unchanged. + let mut trimmed = record("x"); + trimmed.raw_xml = String::new(); + + let with_xml = export_records(&[record("x")], ExportFormat::Csv).expect("exports"); + let without = export_records(&[trimmed], ExportFormat::Csv).expect("exports"); + assert_eq!(with_xml, without); + } + + #[test] + fn a_record_missing_raw_xml_still_deserializes() { + let json = serde_json::json!([{ + "id": 0, "eventRecordId": 1, "timestamp": "", "timestampEpoch": 0, + "provider": "P", "channel": "C", "eventId": 1, "level": "Error", + "computer": "TESTHOST-01", "message": "m", "sourceLabel": "Live", "mapped": [] + }]); + let records: Vec = + serde_json::from_value(json).expect("a trimmed payload deserializes"); + assert_eq!(records[0].raw_xml, ""); + assert!(records[0].event_data.is_empty()); + } + + #[test] + fn extensions_match_the_format() { + assert_eq!(ExportFormat::Csv.extension(), "csv"); + assert_eq!(ExportFormat::Tsv.extension(), "tsv"); + assert_eq!(ExportFormat::Json.extension(), "json"); + assert_eq!(ExportFormat::Xml.extension(), "xml"); + } +} diff --git a/src-tauri/src/event_log/fetch.rs b/src-tauri/src/event_log/fetch.rs new file mode 100644 index 000000000..d9a361431 --- /dev/null +++ b/src-tauri/src/event_log/fetch.rs @@ -0,0 +1,121 @@ +//! What to do when a batched fetch from the Event Log service fails. +//! +//! The decision lives here, apart from the call that produces the error, because everything around +//! it is Windows-only and therefore untestable anywhere else. Getting this wrong is expensive in a +//! specific way: the previous behaviour treated every failure as the end of the channel, so a +//! service that refused one request returned a partial channel that the caller reported as whole. +//! That is not a crash, it is a quiet wrong answer, and only a test that can run everywhere will +//! keep catching it. + +/// Windows `ERROR_NO_MORE_ITEMS`: the channel is exhausted. Not a failure. +const NO_MORE_ITEMS: u32 = 259; + +/// Windows `RPC_S_INVALID_BOUND`: the service refused the size of the request. +/// +/// Seen from `EvtNext` with a 256-handle batch, on one channel out of roughly twelve hundred on a +/// real machine. It says nothing about the channel's contents, so a smaller request is the right +/// answer rather than giving up on the channel. +const INVALID_BOUND: u32 = 1734; + +/// How a failed fetch should be handled. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FetchFailure { + /// The channel is fully read. Stop, and report nothing missing. + Exhausted, + /// Retry this fetch with a smaller batch. + RetryWith(usize), + /// Stop, and report that the channel was read only in part. + Truncated, +} + +/// Decides how to respond to a failed fetch of `batch` events. +/// +/// `floor` is the smallest batch worth attempting. Below it the request is as small as it is going +/// to get, and a further failure is about the channel rather than the request size. +pub fn classify_fetch_failure(win32_code: u32, batch: usize, floor: usize) -> FetchFailure { + if win32_code == NO_MORE_ITEMS { + return FetchFailure::Exhausted; + } + if win32_code == INVALID_BOUND && batch > floor { + // Halved rather than dropped straight to the floor, so a channel that can serve 128 is not + // read at 8 for the rest of the scan. + return FetchFailure::RetryWith((batch / 2).max(floor)); + } + FetchFailure::Truncated +} + +#[cfg(test)] +mod tests { + use super::*; + + const FLOOR: usize = 8; + + #[test] + fn an_exhausted_channel_is_not_a_failure() { + assert_eq!( + classify_fetch_failure(NO_MORE_ITEMS, 256, FLOOR), + FetchFailure::Exhausted + ); + } + + #[test] + fn a_refused_batch_size_is_retried_smaller() { + assert_eq!( + classify_fetch_failure(INVALID_BOUND, 256, FLOOR), + FetchFailure::RetryWith(128) + ); + } + + #[test] + fn halving_stops_at_the_floor_rather_than_reaching_zero() { + // A batch of zero would ask for no events and loop forever without reading anything. + assert_eq!( + classify_fetch_failure(INVALID_BOUND, 9, FLOOR), + FetchFailure::RetryWith(FLOOR) + ); + } + + #[test] + fn a_refusal_at_the_floor_is_reported_as_truncation() { + // The request is already as small as it gets, so the problem is not its size. Reporting + // Exhausted here would present a partly read channel as a complete one. + assert_eq!( + classify_fetch_failure(INVALID_BOUND, FLOOR, FLOOR), + FetchFailure::Truncated + ); + } + + #[test] + fn any_other_error_truncates_rather_than_looking_like_the_end_of_the_channel() { + // ERROR_ACCESS_DENIED partway through a channel is the case that matters: the events after + // it are missing, and calling that "exhausted" is the silent wrong answer. + for code in [5u32, 87, 1500, 0] { + assert_eq!( + classify_fetch_failure(code, 256, FLOOR), + FetchFailure::Truncated, + "win32 {code} must not be mistaken for the end of the channel" + ); + } + } + + #[test] + fn repeated_halving_walks_down_to_the_floor_and_then_stops() { + // The loop this models must terminate. Following the decisions from a full batch has to + // reach Truncated in a bounded number of steps rather than retrying forever. + let mut batch = 256; + let mut steps = 0; + loop { + match classify_fetch_failure(INVALID_BOUND, batch, FLOOR) { + FetchFailure::RetryWith(next) => { + assert!(next < batch, "a retry must shrink the request"); + batch = next; + } + FetchFailure::Truncated => break, + FetchFailure::Exhausted => panic!("a refusal is not an exhausted channel"), + } + steps += 1; + assert!(steps < 20, "halving should reach the floor quickly"); + } + assert_eq!(batch, FLOOR); + } +} diff --git a/src-tauri/src/event_log/live.rs b/src-tauri/src/event_log/live.rs index 7835b9c44..b5123a5b8 100644 --- a/src-tauri/src/event_log/live.rs +++ b/src-tauri/src/event_log/live.rs @@ -1,20 +1,55 @@ use std::collections::HashMap; use std::ffi::c_void; -use std::sync::OnceLock; -use regex::Regex; - -use super::models::{ChannelSourceType, EvtxChannelInfo, EvtxField, EvtxLevel, EvtxRecord}; -use super::sanitize_control_chars; +use super::event_node::{extract_system_fields, parse_event_xml}; +use super::models::{ChannelSourceType, EvtxChannelInfo, EvtxRecord}; +use cmtraceopen_parser::event_query::{build_query, EventQueryFilter}; +use cmtraceopen_parser::eventmap::MapRegistry; #[cfg(target_os = "windows")] use windows::core::{Error, HSTRING, PCWSTR}; #[cfg(target_os = "windows")] use windows::Win32::System::EventLog::{ EvtClose, EvtFormatMessage, EvtFormatMessageEvent, EvtNext, EvtOpenPublisherMetadata, EvtQuery, - EvtQueryChannelPath, EvtQueryReverseDirection, EvtRender, EvtRenderEventXml, EVT_HANDLE, + EvtQueryChannelPath, EvtQueryReverseDirection, EvtQueryTolerateQueryErrors, EvtRender, + EvtRenderEventXml, EVT_HANDLE, }; +/// Event handles fetched per `EvtNext` call. +/// +/// Each call is a round trip to the Event Log service, so this is the dominant cost of a scan. +/// FullEventLogView hardcodes 1, paying one round trip per event. The API accepts up to 1024; +/// 256 keeps the per-call array modest while cutting round trips by that factor. +#[cfg(target_os = "windows")] +const EVENT_FETCH_BATCH: usize = 256; + +/// Smallest batch to fall back to before treating the channel as unreadable. +/// +/// Some channels reject a 256-handle request with `RPC_S_INVALID_BOUND`. Measuring a full scan +/// found one doing exactly that, and the loop's response was to stop reading and return what it +/// already had as a complete result. Halving down to this floor reads the channel instead. +#[cfg(target_os = "windows")] +const MIN_FETCH_BATCH: usize = 8; + +/// What one channel yielded, including why anything is missing from it. +/// +/// The records used to be returned on their own, which left no way to say "this channel was read, +/// but not all of it". A partial read then reported as a complete one: the caller saw `Ok`, counted +/// the events, and showed a channel that looked fully loaded. Events that were never fetched are +/// indistinguishable on screen from events that do not exist. +pub struct ChannelScan { + /// Records the caller did not take. Empty for a caller that streamed every batch away. + pub records: Vec, + /// How many records this channel produced in total, whether or not the caller kept them. + /// + /// Separate from `records.len()` because a streaming caller empties that vector as it goes. + /// Reporting the length instead would tell the frontend a fully read channel held no events, + /// which is the same wrong answer as a channel that failed. + pub delivered: usize, + /// Operator-facing explanations of what is missing. Empty means the channel was read whole. + pub gaps: Vec, +} + // ── RAII handle wrapper ───────────────────────────────────────────────────── #[cfg(target_os = "windows")] @@ -113,29 +148,116 @@ pub fn enumerate_channels() -> Result, String> { /// Query events from a live Windows Event Log channel. /// -/// Returns newest events first, capped at `max_events` (default 1000). +/// Returns newest events first. `None` means no cap, which is what every caller in the application +/// passes; there is no default limit, and the comment claiming a default of 1000 described a +/// behaviour this function has not had. A cap that is documented but absent is worse than either, +/// because it invites callers to rely on a bound nothing enforces. #[cfg(target_os = "windows")] -pub fn query_channel(channel: &str, max_events: Option) -> Result, String> { - query_channel_with_progress(channel, max_events, |_, _| {}) +pub fn query_channel( + channel: &str, + maps: &MapRegistry, + max_events: Option, +) -> Result { + query_channel_with_progress(channel, maps, max_events, |_, _| {}) +} + +/// Queries a channel with server-side filtering. +/// +/// The filter is compiled to XPath and evaluated by the service, so events that do not match are +/// never fetched, rendered, or transferred. +#[cfg(target_os = "windows")] +pub fn query_channel_filtered( + channel: &str, + filter: &EventQueryFilter, + maps: &MapRegistry, + max_events: Option, +) -> Result { + query_channel_inner(channel, filter, maps, max_events, |_, _| {}, |_| {}) } /// Query with a progress callback: `on_progress(fetched_so_far, total_estimate)`. #[cfg(target_os = "windows")] pub fn query_channel_with_progress( channel: &str, + maps: &MapRegistry, max_events: Option, on_progress: impl Fn(usize, Option), -) -> Result, String> { +) -> Result { + query_channel_inner( + channel, + &EventQueryFilter::default(), + maps, + max_events, + on_progress, + |_| {}, + ) +} + +/// Queries a channel with server-side filtering, reporting progress as events arrive. +#[cfg(target_os = "windows")] +pub fn query_channel_filtered_with_progress( + channel: &str, + filter: &EventQueryFilter, + maps: &MapRegistry, + max_events: Option, + on_progress: impl Fn(usize, Option), +) -> Result { + query_channel_inner(channel, filter, maps, max_events, on_progress, |_| {}) +} + +/// Queries a channel, delivering each batch of records as it is read. +/// +/// `on_batch` is handed every batch and is expected to take the records from it. Whatever it leaves +/// is returned in the [`ChannelScan`], so a caller that forgets to drain still gets correct results +/// rather than losing them; it simply holds the channel in memory as before. +#[cfg(target_os = "windows")] +pub fn query_channel_streamed( + channel: &str, + filter: &EventQueryFilter, + maps: &MapRegistry, + max_events: Option, + on_progress: impl Fn(usize, Option), + on_batch: impl FnMut(&mut Vec), +) -> Result { + query_channel_inner(channel, filter, maps, max_events, on_progress, on_batch) +} + +/// Reads a channel, handing each fetched batch to `on_batch` as it is built. +/// +/// `on_batch` receives the batch by mutable reference and may take the records out of it. Whatever +/// it leaves behind is accumulated into the returned [`ChannelScan`]. That is the whole difference +/// between streaming and collecting: a caller that drains never holds more than one batch, and a +/// caller that ignores the argument gets the channel in one piece exactly as before. +/// +/// The distinction matters because one channel dominates a scan. On a measured seven-day scan, +/// Security was 286,401 of 404,769 events and 191.8 seconds of 267, so a caller waiting for this +/// function to return waits three minutes with nothing to show. +#[cfg(target_os = "windows")] +fn query_channel_inner( + channel: &str, + filter: &EventQueryFilter, + maps: &MapRegistry, + max_events: Option, + on_progress: impl Fn(usize, Option), + mut on_batch: impl FnMut(&mut Vec), +) -> Result { let limit = max_events.map(|n| n as usize).unwrap_or(usize::MAX); let channel_hstring = HSTRING::from(channel); - let query_string = HSTRING::from("*"); + // A filter that cannot be expressed is refused here rather than silently degraded to "*", + // which would return everything and look like the filter simply matched a lot. + let compiled = build_query(filter) + .map_err(|error| format!("cannot compile event query for {channel}: {error}"))?; + let query_string = HSTRING::from(compiled.as_str()); let query_handle = unsafe { EvtQuery( None, &channel_hstring, &query_string, - EvtQueryChannelPath.0 | EvtQueryReverseDirection.0, + // TolerateQueryErrors keeps a scan alive when one part of a query cannot be evaluated, + // for example a provider that is not registered on this machine. Without it a single + // bad element aborts the whole channel and the result silently looks empty. + EvtQueryChannelPath.0 | EvtQueryReverseDirection.0 | EvtQueryTolerateQueryErrors.0, ) } .map_err(|e| format_error(&format!("EvtQuery({channel})"), &e))?; @@ -144,23 +266,62 @@ pub fn query_channel_with_progress( let mut records = Vec::new(); let mut publisher_metadata = HashMap::>::new(); - - while records.len() < limit { - let mut raw_handles = [0isize; 16]; + let mut unparsable = 0usize; + let mut unrenderable = 0usize; + + let mut gaps = Vec::new(); + let mut batch = EVENT_FETCH_BATCH; + // Counted separately from `records`, which a streaming caller empties as it goes. Using the + // length of a vector the caller is allowed to drain would restart the limit at zero after every + // batch and read the channel forever. + let mut produced = 0usize; + + while produced < limit { + let mut raw_handles = [0isize; EVENT_FETCH_BATCH]; let mut returned = 0u32; - match unsafe { EvtNext(query_handle.raw(), &mut raw_handles, 0, 0, &mut returned) } { - Ok(()) => {} - Err(e) => { - if !is_no_more_items(&e) { - eprintln!( - "[evtx] EvtNext error: code=0x{:08x} w32={} msg=\"{}\"", - e.code().0 as u32, - win32_code(&e), - e.message() + let fetched = unsafe { + EvtNext( + query_handle.raw(), + &mut raw_handles[..batch], + 0, + 0, + &mut returned, + ) + }; + + if let Err(error) = fetched { + // How to respond is decided in `super::fetch`, where it is tested on every platform. + // Everything else in this loop is Windows-only, so a rule encoded here is a rule CI + // cannot check on any runner. + match super::fetch::classify_fetch_failure(win32_code(&error), batch, MIN_FETCH_BATCH) { + super::fetch::FetchFailure::Exhausted => break, + super::fetch::FetchFailure::RetryWith(smaller) => { + batch = smaller; + log::info!( + "event=evtx_batch_reduced channel=\"{channel}\" batch={batch} \ + reason=\"the service rejected the previous batch size\"" ); + continue; + } + super::fetch::FetchFailure::Truncated => { + // Recorded as a gap, not only logged. The records already read are still + // returned because they are real, but the channel must not be presented as + // complete when the rest of it was never fetched. + log::warn!( + "event=evtx_next_failed channel=\"{channel}\" batch={batch} \ + w32={} code=0x{:08x}", + win32_code(&error), + error.code().0 as u32 + ); + gaps.push(format!( + "{channel}: stopped after {} events, the channel could not be read further ({}, 0x{:08x})", + produced, + error.message().trim(), + error.code().0 as u32 + )); + break; } - break; } } @@ -168,8 +329,14 @@ pub fn query_channel_with_progress( break; } + // Built per fetch rather than appended straight to `records`, so the caller can take each + // batch as it is produced. A caller that takes them holds nothing here, which is what keeps + // a channel the size of Security from occupying its whole result set before anything is + // shown. + let mut batch_records: Vec = Vec::new(); + for raw_handle in raw_handles.into_iter().take(returned as usize) { - if records.len() >= limit { + if produced + batch_records.len() >= limit { // Close remaining handles we won't use unsafe { let _ = EvtClose(EVT_HANDLE(raw_handle)); @@ -178,39 +345,97 @@ pub fn query_channel_with_progress( } let event_handle = OwnedEvtHandle::new(EVT_HANDLE(raw_handle)); - let xml = - render_event_xml(event_handle.raw()).map_err(|e| format_error("EvtRender", &e))?; + // A handle that fails to render is counted, not fatal. Propagating it here returned + // Err for the whole channel and threw away every record already read, which the caller + // then reported as a channel with no events. + let xml = match render_event_xml(event_handle.raw()) { + Ok(xml) => xml, + Err(error) => { + unrenderable += 1; + if unrenderable == 1 { + log::warn!( + "event=evtx_render_failed channel=\"{channel}\" error=\"{}\"", + format_error("EvtRender", &error) + ); + } + continue; + } + }; - let provider_name = extract_xml_attr(&xml, "Provider", "Name"); + // Parsed once here and handed to the record builder. The provider has to be known + // before the record exists, because it names the publisher whose message template the + // service is asked to render, and parsing a second time to learn the rest would double + // the cost of the hottest loop in this view. + let parsed = match parse_event_xml(&xml) { + Ok(parsed) => parsed, + Err(error) => { + unparsable += 1; + if unparsable == 1 { + // Sliced by character rather than by byte. A byte cut that lands inside a + // multi-byte character panics, and this XML carries account names and paths + // that are routinely not ASCII. + let prefix: String = xml.chars().take(300).collect(); + log::warn!( + "event=evtx_parse_failed channel=\"{channel}\" error=\"{error}\" xml_prefix=\"{prefix}\"" + ); + } + continue; + } + }; + let system = extract_system_fields(&parsed); - // Try to get a formatted message via EvtFormatMessage - let rendered_message = provider_name.as_deref().and_then(|provider| { + // Only attempted when the event named a provider. Asking the service for the metadata + // of a publisher the event never named would fail once per event and cache the failure + // under a name no provider has. + let rendered_message = system.provider.as_deref().and_then(|provider| { format_event_message(event_handle.raw(), provider, &mut publisher_metadata) .ok() .flatten() }); - if let Some(record) = parse_xml_to_record(&xml, channel, rendered_message.as_deref()) { - records.push(record); - // Report progress every 100 records - if records.len() % 100 == 0 { - on_progress(records.len(), None); - } - } else if records.is_empty() { - // Log the first unparseable XML so we can debug the format - log::warn!( - "event=evtx_parse_failed channel=\"{channel}\" xml_prefix=\"{}\"", - &xml[..xml.len().min(300)] - ); - } + batch_records.push(super::rendered::record_from_parts( + &parsed, + system, + &xml, + channel, + maps, + rendered_message.as_deref(), + )); } + + produced += batch_records.len(); + on_progress(produced, None); + + // The caller sees the batch before anything else happens to it. Draining it here is what + // makes delivery incremental; leaving it collects the channel as before. + on_batch(&mut batch_records); + records.append(&mut batch_records); } + if unparsable > 0 { + // Counted and reported rather than passed over. Events that never arrived look exactly like + // evidence that the thing being investigated did not happen. + log::warn!("event=evtx_live_query_gap channel=\"{channel}\" unparsable={unparsable}"); + gaps.push(format!( + "{channel}: {unparsable} events could not be read and are missing from this view" + )); + } + if unrenderable > 0 { + log::warn!("event=evtx_live_query_gap channel=\"{channel}\" unrenderable={unrenderable}"); + gaps.push(format!( + "{channel}: {unrenderable} events could not be rendered and are missing from this view" + )); + } log::info!( - "event=evtx_live_query_done channel=\"{channel}\" records={}", - records.len() + "event=evtx_live_query_done channel=\"{channel}\" records={} unparsable={unparsable} unrenderable={unrenderable} gaps={}", + records.len(), + gaps.len() ); - Ok(records) + Ok(ChannelScan { + records, + delivered: produced, + gaps, + }) } // ── Non-Windows stubs ─────────────────────────────────────────────────────── @@ -223,14 +448,40 @@ pub fn enumerate_channels() -> Result, String> { #[cfg(not(target_os = "windows"))] pub fn query_channel_with_progress( _channel: &str, + _maps: &MapRegistry, _max_events: Option, _on_progress: impl Fn(usize, Option), -) -> Result, String> { +) -> Result { Err("Live event log queries are only available on Windows.".to_string()) } #[cfg(not(target_os = "windows"))] -pub fn query_channel(_channel: &str, _max_events: Option) -> Result, String> { +pub fn query_channel( + _channel: &str, + _maps: &MapRegistry, + _max_events: Option, +) -> Result { + Err("Live event log queries are only available on Windows.".to_string()) +} + +#[cfg(not(target_os = "windows"))] +pub fn query_channel_filtered( + _channel: &str, + _filter: &EventQueryFilter, + _maps: &MapRegistry, + _max_events: Option, +) -> Result { + Err("Live event log queries are only available on Windows.".to_string()) +} + +#[cfg(not(target_os = "windows"))] +pub fn query_channel_filtered_with_progress( + _channel: &str, + _filter: &EventQueryFilter, + _maps: &MapRegistry, + _max_events: Option, + _on_progress: impl Fn(usize, Option), +) -> Result { Err("Live event log queries are only available on Windows.".to_string()) } @@ -320,153 +571,6 @@ fn format_event_message( } } -// ── XML parsing helpers ───────────────────────────────────────────────────── - -/// Parse rendered event XML into an EvtxRecord. -fn parse_xml_to_record( - xml: &str, - channel: &str, - rendered_message: Option<&str>, -) -> Option { - let event_id_str = extract_xml_text(xml, "EventID").unwrap_or_default(); - let event_id: u32 = event_id_str.parse().unwrap_or(0); - - let level_str = extract_xml_text(xml, "Level").unwrap_or_default(); - let level_val: u8 = level_str.parse().unwrap_or(4); - let level = EvtxLevel::from_level_value(level_val); - - let provider = extract_xml_attr(xml, "Provider", "Name").unwrap_or_default(); - let computer = extract_xml_text(xml, "Computer").unwrap_or_default(); - let event_record_id: u64 = extract_xml_text(xml, "EventRecordID") - .and_then(|s| s.parse().ok()) - .unwrap_or(0); - - let timestamp = extract_xml_attr(xml, "TimeCreated", "SystemTime").unwrap_or_default(); - let timestamp_epoch = parse_timestamp_to_epoch_ms(×tamp); - - let event_data = extract_xml_event_data(xml); - - // Use rendered message if available, otherwise build summary from EventData. - // Sanitize to strip control characters that would show as unexpected glyphs. - let message = rendered_message - .map(sanitize_control_chars) - .unwrap_or_else(|| build_event_data_summary(&event_data)); - - Some(EvtxRecord { - id: 0, // assigned by commands.rs after sorting - event_record_id, - timestamp, - timestamp_epoch, - provider, - channel: channel.to_string(), - event_id, - level, - computer, - message, - event_data, - raw_xml: xml.to_string(), - source_label: "Live".to_string(), - }) -} - -/// Extract an attribute value from an XML tag. -/// e.g. `extract_xml_attr(xml, "Provider", "Name")` for `` -fn extract_xml_attr(xml: &str, tag: &str, attr: &str) -> Option { - let tag_start = xml.find(&format!("<{tag}"))?; - let tag_end = xml[tag_start..].find('>')? + tag_start; - let tag_content = &xml[tag_start..=tag_end]; - - // Try both quote styles: Name='value' and Name="value" - for quote in ['"', '\''] { - let pattern = format!("{attr}={quote}"); - if let Some(attr_start) = tag_content.find(&pattern) { - let value_start = attr_start + pattern.len(); - if let Some(value_end) = tag_content[value_start..].find(quote) { - return Some(tag_content[value_start..value_start + value_end].to_string()); - } - } - } - None -} - -/// Extract text content between XML tags. -/// e.g. `extract_xml_text(xml, "EventID")` for `123` -fn extract_xml_text(xml: &str, tag: &str) -> Option { - let open = format!("<{tag}>"); - let open_with_attrs = format!("<{tag} "); - let close = format!(""); - - // Try simple value - if let Some(start) = xml.find(&open) { - let value_start = start + open.len(); - if let Some(end) = xml[value_start..].find(&close) { - return Some(xml[value_start..value_start + end].trim().to_string()); - } - } - - // Try value - if let Some(start) = xml.find(&open_with_attrs) { - let after_tag = &xml[start..]; - let tag_close = after_tag.find('>')?; - let value_start = start + tag_close + 1; - if let Some(end) = xml[value_start..].find(&close) { - return Some(xml[value_start..value_start + end].trim().to_string()); - } - } - - None -} - -/// Extract `value` pairs from EventData section. -/// -/// All values are sanitized to strip control characters (e.g. `\r`, `\0`) that -/// would render as unexpected glyphs in the UI. -fn extract_xml_event_data(xml: &str) -> Vec { - fn data_name_re() -> &'static Regex { - static CELL: OnceLock = OnceLock::new(); - CELL.get_or_init(|| { - Regex::new(r#"(.*?)"#).expect("data regex") - }) - } - - data_name_re() - .captures_iter(xml) - .map(|cap| EvtxField { - name: cap[1].to_string(), - value: sanitize_control_chars(&cap[2]), - }) - .collect() -} - -/// Build a summary message from EventData fields (fallback when EvtFormatMessage unavailable). -fn build_event_data_summary(fields: &[EvtxField]) -> String { - fields - .iter() - .take(5) - .map(|f| { - let val = if f.value.len() > 80 { - format!("{}...", &f.value[..77]) - } else { - f.value.clone() - }; - format!("{}: {val}", f.name) - }) - .collect::>() - .join("; ") -} - -/// Parse an ISO 8601 timestamp to epoch milliseconds. -fn parse_timestamp_to_epoch_ms(timestamp: &str) -> i64 { - chrono::DateTime::parse_from_rfc3339(timestamp) - .or_else(|_| { - // Windows timestamps may omit timezone, assume UTC - chrono::NaiveDateTime::parse_from_str(timestamp, "%Y-%m-%dT%H:%M:%S%.f") - .map(|naive| naive.and_utc().fixed_offset()) - }) - .map(|dt| dt.timestamp_millis()) - .unwrap_or(0) -} - // ── Error helpers ─────────────────────────────────────────────────────────── #[cfg(target_os = "windows")] @@ -490,10 +594,8 @@ fn is_insufficient_buffer(error: &Error) -> bool { win32_code(error) == 122 } -#[cfg(target_os = "windows")] -fn is_no_more_items(error: &Error) -> bool { - win32_code(error) == 259 -} +// `ERROR_NO_MORE_ITEMS` and `RPC_S_INVALID_BOUND` are recognised in `super::fetch`, which owns the +// decision they feed and is tested on every platform rather than only on this one. #[cfg(target_os = "windows")] fn is_not_found(error: &Error) -> bool { @@ -517,7 +619,9 @@ mod tests { let has_app = channels.iter().any(|c| c.name == "Application"); println!("Has Application channel: {has_app}"); - let records = query_channel("Application", Some(3)).expect("query should work"); + let records = query_channel("Application", &MapRegistry::new(), Some(3)) + .expect("query should work") + .records; println!("Application records: {}", records.len()); for (i, r) in records.iter().enumerate() { println!("--- Record {i} ---"); @@ -531,3 +635,163 @@ mod tests { } } } + +#[cfg(all(test, target_os = "windows"))] +mod live_service_tests { + //! Exercises the live path against the machine's own Event Log service. + //! + //! Ignored by default because it needs a real service with real events, which CI runners and + //! developer machines cannot be relied on to have. Run deliberately on a Windows host: + //! + //! ```text + //! cargo test --lib event_log::live::live_service_tests -- --ignored --nocapture + //! ``` + //! + //! These exist because every assumption in this file that was checked only by reasoning turned + //! out to be wrong at least once. Compilation proves nothing about whether the service accepts + //! what we send it. + + use super::*; + // Only the assertions need the level type; the query path itself no longer builds records, so + // importing it at module scope would warn in a non-test build. + use super::super::models::EvtxLevel; + use cmtraceopen_parser::event_query::{EventQueryFilter, TimeWindow}; + + const CHANNEL: &str = "Application"; + + /// A registry local to the call, replacing what used to be an implicit process global. These + /// tests are about the query reaching the service, not about mapping. + fn no_maps() -> MapRegistry { + MapRegistry::new() + } + + #[test] + #[ignore = "requires a live Windows Event Log service with events"] + fn an_unfiltered_query_returns_records() { + let scan = query_channel(CHANNEL, &no_maps(), Some(50)).expect("query succeeds"); + let records = scan.records; + assert!( + !records.is_empty(), + "Application channel should have events" + ); + let first = &records[0]; + assert!(!first.provider.is_empty(), "provider should be populated"); + assert_eq!(first.channel, CHANNEL); + assert!(first.event_id > 0); + } + + #[test] + #[ignore = "requires a live Windows Event Log service with events"] + fn a_time_filter_is_applied_by_the_service_and_narrows_the_result() { + // The narrow window is queried first. Running the wide one first leaves a gap in which a + // newly written event lands inside the one-hour result and outside the thirty-day one + // already collected, failing the assertion for a reason that has nothing to do with + // filtering. Querying narrow first makes any new event land in the wide result, which is + // the direction the assertion tolerates. + let narrow = query_channel_filtered( + CHANNEL, + &EventQueryFilter { + time: Some(TimeWindow::Last { + milliseconds: 60 * 60 * 1000, + }), + ..Default::default() + }, + &no_maps(), + None, + ) + .expect("1 hour query succeeds") + .records; + + let wide = query_channel_filtered( + CHANNEL, + &EventQueryFilter { + time: Some(TimeWindow::Last { + milliseconds: 30 * 24 * 60 * 60 * 1000, + }), + ..Default::default() + }, + &no_maps(), + None, + ) + .expect("30 day query succeeds") + .records; + + assert!( + narrow.len() <= wide.len(), + "a narrower window cannot return more events: {} vs {}", + narrow.len(), + wide.len() + ); + } + + #[test] + #[ignore = "requires a live Windows Event Log service with events"] + fn a_level_filter_returns_only_that_level() { + // Level 2 is Error. If the predicate were dropped or malformed the service would either + // reject the query or return everything, and both show up here. + let records = query_channel_filtered( + CHANNEL, + &EventQueryFilter { + levels: vec![2], + ..Default::default() + }, + &no_maps(), + Some(200), + ) + .expect("level query succeeds") + .records; + + for record in &records { + assert_eq!( + record.level, + EvtxLevel::Error, + "level filter must be applied by the service, got {:?}", + record.level + ); + } + } + + #[test] + #[ignore = "requires a live Windows Event Log service with events"] + fn an_impossible_filter_returns_nothing_rather_than_everything() { + // A malformed predicate that the service ignores would show up as a full result set. + let records = query_channel_filtered( + CHANNEL, + &EventQueryFilter { + event_ids: vec![cmtraceopen_parser::event_query::EventIdSelector::Single { + id: 999_999, + }], + ..Default::default() + }, + &no_maps(), + Some(50), + ) + .expect("query succeeds") + .records; + + assert!( + records.is_empty(), + "event id 999999 should match nothing, got {}", + records.len() + ); + } + + #[test] + #[ignore = "requires a live Windows Event Log service with events"] + fn system_fields_are_populated_from_real_events() { + let records = + query_channel_filtered(CHANNEL, &EventQueryFilter::default(), &no_maps(), Some(200)) + .expect("query succeeds") + .records; + + assert!(!records.is_empty()); + assert!( + records.iter().any(|r| r.process_id.is_some()), + "at least one real event should carry Execution/@ProcessID" + ); + assert!( + records.iter().any(|r| r.keywords.is_some()), + "at least one real event should carry Keywords" + ); + } +} diff --git a/src-tauri/src/event_log/maps.rs b/src-tauri/src/event_log/maps.rs new file mode 100644 index 000000000..d4a5f79e7 --- /dev/null +++ b/src-tauri/src/event_log/maps.rs @@ -0,0 +1,542 @@ +//! Loading EvtxECmd `.map` files from disk. +//! +//! The schema and the engine live in `cmtraceopen-parser`, which stays pure and +//! wasm32-compatible and therefore has no YAML dependency. This module is the host-side adapter: +//! it reads `.map` files, deserializes the YAML, and builds a +//! [`MapRegistry`](cmtraceopen_parser::eventmap::MapRegistry). +//! +//! Two behaviours were verified against EvtxECmd 1.5.2 on a Windows 11 host rather than inferred +//! from its documentation: +//! +//! - **First loaded wins.** Files load in alphabetical order and a later file with the same +//! identity is rejected, which is what makes the documented `1_` prefix override work. EvtxECmd +//! reports this as `An item with the same key has already been added. Key: 326-APPLICATION-ESENT` +//! and continues with the first map. +//! - **Identity is case-insensitive.** That same key is uppercased channel and provider, matching +//! how `MapRegistry` compares them. + +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use cmtraceopen_parser::eventmap::{apply_map, EventMap, EventNode, MapProperty, MapRegistry}; +use serde::{Deserialize, Serialize}; + +/// Why a `.map` file could not be used. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct MapLoadFailure { + /// The file that failed. + pub path: String, + /// The reason, suitable for showing to an operator. + pub reason: String, +} + +/// A map that parsed but lost to an earlier file claiming the same identity. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct SupersededMap { + /// The file that was skipped. + pub path: String, + /// The file that already owned this identity. + pub superseded_by: String, + /// `channel/provider/eventId` of the contested identity. + pub identity: String, +} + +/// The result of loading a directory of maps. +/// +/// Failures and supersessions are reported rather than silently dropped: a map that did not load +/// means events of that type render unmapped, which is a coverage gap an operator needs to see. +#[derive(Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MapLoadOutcome { + /// Files that loaded and won their identity. + pub loaded: Vec, + /// Files skipped because an earlier file already owned the identity. + pub superseded: Vec, + /// Files that could not be read or parsed. + pub failures: Vec, +} + +impl MapLoadOutcome { + /// Number of maps actually registered. + pub fn loaded_count(&self) -> usize { + self.loaded.len() + } + + /// True when nothing failed and nothing was skipped. + pub fn is_clean(&self) -> bool { + self.failures.is_empty() && self.superseded.is_empty() + } +} + +/// Parses one `.map` file's contents. +/// +/// Upstream maps are UTF-8 with a byte order mark, which YAML parsers reject as an unexpected +/// character, so the BOM is stripped first. +pub fn parse_map(contents: &str) -> Result { + let without_bom = contents.strip_prefix('\u{feff}').unwrap_or(contents); + serde_norway::from_str::(without_bom).map_err(|error| error.to_string()) +} + +/// Reads a map file as text, falling back to Windows-1252 when it is not UTF-8. +/// +/// The upstream corpus is UTF-8, but a map written by hand on a Windows machine often is not, and +/// `read_to_string` would reject it with "stream did not contain valid UTF-8". That reads as a +/// corrupt file when it is really an encoding the rest of this codebase already handles, so the +/// same UTF-8 then Windows-1252 fallback used for log files applies here. +fn read_map_file(path: &Path) -> Result { + let bytes = fs::read(path).map_err(|error| error.to_string())?; + match String::from_utf8(bytes) { + Ok(text) => Ok(text), + Err(error) => { + let (text, _, _) = encoding_rs::WINDOWS_1252.decode(error.as_bytes()); + Ok(text.into_owned()) + } + } +} + +/// Load order for two map files. +/// +/// Case-insensitive first, so a `1_`-prefixed copy wins regardless of case, which is what makes +/// the documented override work. The exact name breaks ties: without it `Map.map` and `map.map` +/// compare equal, and on a case-sensitive filesystem which one claims the identity would depend on +/// the unspecified order `read_dir` happened to return them in. Since load order decides which map +/// wins, that would make the registry differ between machines holding identical directories. +fn map_file_order(left: &Path, right: &Path) -> std::cmp::Ordering { + let name_of = |path: &Path| { + path.file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default() + .to_string() + }; + let (left_name, right_name) = (name_of(left), name_of(right)); + left_name + .to_ascii_lowercase() + .cmp(&right_name.to_ascii_lowercase()) + .then_with(|| left_name.cmp(&right_name)) +} + +fn identity_of(map: &EventMap) -> String { + format!("{}/{}/{}", map.channel, map.provider, map.event_id) +} + +/// Loads every `.map` file directly inside `directory`. +/// +/// Files are processed in case-insensitive alphabetical order so a `1_`-prefixed copy wins, which +/// is how upstream lets a local customization survive a map-corpus update. +pub fn load_maps_from_dir(directory: &Path) -> Result<(MapRegistry, MapLoadOutcome), String> { + let entries = fs::read_dir(directory) + .map_err(|error| format!("cannot read map directory {}: {error}", directory.display()))?; + + let mut registry = MapRegistry::new(); + let mut outcome = MapLoadOutcome::default(); + + let mut files: Vec = Vec::new(); + for entry in entries { + // An enumeration error after the directory itself opened is recorded rather than skipped. + // Dropping it would let a partial registry come back with is_clean() true, which reads as + // "every map loaded" when a map is in fact missing and its columns are silently absent. + match entry { + Ok(entry) => files.push(entry.path()), + Err(error) => outcome.failures.push(MapLoadFailure { + path: directory.display().to_string(), + reason: format!("cannot read directory entry: {error}"), + }), + } + } + files.retain(|path| { + path.is_file() + && path + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("map")) + }); + files.sort_by(|left, right| map_file_order(left, right)); + // identity (lowercased, matching MapRegistry's comparison) -> the file that claimed it + let mut owners: HashMap = HashMap::new(); + + for path in files { + let display = path.display().to_string(); + let contents = match read_map_file(&path) { + Ok(contents) => contents, + Err(error) => { + outcome.failures.push(MapLoadFailure { + path: display, + reason: format!("cannot read file: {error}"), + }); + continue; + } + }; + + let map = match parse_map(&contents) { + Ok(map) => map, + Err(reason) => { + outcome.failures.push(MapLoadFailure { + path: display, + reason, + }); + continue; + } + }; + + // First loaded wins, matching EvtxECmd. Checking before inserting is deliberate: + // MapRegistry::insert is last-wins, which is the opposite of what upstream does. + let identity = identity_of(&map); + let owner_key = identity.to_ascii_lowercase(); + if let Some(owner) = owners.get(&owner_key) { + outcome.superseded.push(SupersededMap { + path: display, + superseded_by: owner.clone(), + identity, + }); + continue; + } + + owners.insert(owner_key, display.clone()); + registry.insert(map); + outcome.loaded.push(display); + } + + Ok((registry, outcome)) +} + +#[cfg(test)] +mod tests { + use super::*; + use cmtraceopen_parser::eventmap::{apply_map, EventNode, MapProperty}; + use std::fs; + + /// A real upstream map, byte for byte, including the leading comment style and quoting. + pub(super) const SHELL_CORE_9701: &str = r#"Author: Troy Larson +Description: RunOnceEx commands started +EventId: 9701 +Channel: Microsoft-Windows-Shell-Core/Operational +Provider: Microsoft-Windows-Shell-Core +Maps: + - + Property: PayloadData1 + PropertyValue: "%PayloadData1%" + Values: + - + Name: PayloadData1 + Value: "/Event/EventData/Data" + +# Documentation: +# https://www.geoffchappell.com/notes/windows/shell/events/core.htm +"#; + + fn temp_dir(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("cmtraceopen-maps-{name}")); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("temp dir"); + dir + } + + fn write(dir: &Path, name: &str, contents: &str) { + fs::write(dir.join(name), contents).expect("write map"); + } + + #[test] + fn parses_a_real_upstream_map_including_trailing_comments() { + let map = parse_map(SHELL_CORE_9701).expect("parses"); + assert_eq!(map.event_id, 9701); + assert_eq!(map.provider, "Microsoft-Windows-Shell-Core"); + assert_eq!(map.author.as_deref(), Some("Troy Larson")); + assert_eq!(map.maps.len(), 1); + assert_eq!(map.maps[0].property, MapProperty::PayloadData(1)); + } + + #[test] + fn strips_the_utf8_byte_order_mark_upstream_files_carry() { + let with_bom = format!("\u{feff}{SHELL_CORE_9701}"); + let map = parse_map(&with_bom).expect("BOM-prefixed map parses"); + assert_eq!(map.event_id, 9701); + } + + #[test] + fn reports_a_malformed_map_as_a_failure_rather_than_aborting_the_load() { + let dir = temp_dir("malformed"); + write(&dir, "good.map", SHELL_CORE_9701); + write(&dir, "bad.map", "EventId: [this is not a scalar\nChannel:"); + + let (registry, outcome) = load_maps_from_dir(&dir).expect("directory reads"); + + assert_eq!(registry.len(), 1); + assert_eq!(outcome.loaded_count(), 1); + assert_eq!(outcome.failures.len(), 1); + assert!(outcome.failures[0].path.ends_with("bad.map")); + assert!(!outcome.is_clean()); + } + + #[test] + fn first_loaded_wins_so_a_numeric_prefix_overrides() { + let dir = temp_dir("override"); + let original = SHELL_CORE_9701.replace("%PayloadData1%", "ORIGINAL %PayloadData1%"); + let custom = SHELL_CORE_9701.replace("%PayloadData1%", "CUSTOM %PayloadData1%"); + write(&dir, "Shell-Core_9701.map", &original); + write(&dir, "1_Shell-Core_9701.map", &custom); + + let (registry, outcome) = load_maps_from_dir(&dir).expect("directory reads"); + + assert_eq!(registry.len(), 1); + assert_eq!(outcome.superseded.len(), 1); + assert!(outcome.superseded[0].path.ends_with("Shell-Core_9701.map")); + assert!(outcome.superseded[0].superseded_by.contains("1_")); + + let event = EventNode::new("Event").with_child( + EventNode::new("EventData").with_child(EventNode::new("Data").with_text("cmd.exe")), + ); + let map = registry + .find( + "Microsoft-Windows-Shell-Core/Operational", + "Microsoft-Windows-Shell-Core", + 9701, + ) + .expect("map registered"); + assert_eq!( + apply_map(map, &event).value_for(&MapProperty::PayloadData(1)), + Some("CUSTOM cmd.exe"), + "the 1_-prefixed file loads first and must win" + ); + } + + #[test] + fn a_numeric_prefix_sorts_first_regardless_of_case() { + use std::cmp::Ordering; + assert_eq!( + map_file_order(Path::new("/m/1_Shell.map"), Path::new("/m/Shell.map")), + Ordering::Less + ); + assert_eq!( + map_file_order(Path::new("/m/1_shell.map"), Path::new("/m/Shell.map")), + Ordering::Less + ); + } + + #[test] + fn names_differing_only_in_case_have_a_defined_order() { + use std::cmp::Ordering; + // Tested directly rather than through a directory, because the filesystem this runs on may + // be case-insensitive and would collapse the two names into one file. The ordering is what + // matters: load order decides which map wins, so leaving these equal would let identical + // directories produce different registries on different machines. + assert_eq!( + map_file_order(Path::new("/m/Shell.map"), Path::new("/m/shell.map")), + Ordering::Less, + "uppercase must sort before lowercase, not compare equal" + ); + assert_eq!( + map_file_order(Path::new("/m/shell.map"), Path::new("/m/Shell.map")), + Ordering::Greater, + "the order must be antisymmetric" + ); + assert_eq!( + map_file_order(Path::new("/m/Shell.map"), Path::new("/m/Shell.map")), + Ordering::Equal + ); + } + + #[test] + fn case_insensitive_ordering_still_decides_unrelated_names() { + use std::cmp::Ordering; + assert_eq!( + map_file_order(Path::new("/m/Apple.map"), Path::new("/m/banana.map")), + Ordering::Less, + "case must not dominate the alphabetical comparison" + ); + } + + #[test] + fn a_windows_1252_map_is_read_rather_than_rejected() { + // read_to_string would reject this as invalid UTF-8, which reads as a corrupt file when it + // is really an encoding this codebase already handles everywhere else. + let dir = temp_dir("cp1252"); + let text = SHELL_CORE_9701.replace("Shell-Core 9701", "Shell-Core 9701 caf\u{e9}"); + let mut bytes: Vec = Vec::new(); + for ch in text.chars() { + if ch == '\u{e9}' { + bytes.push(0xE9); // Windows-1252 e-acute, which is not valid UTF-8 on its own + } else { + let mut buffer = [0u8; 4]; + bytes.extend_from_slice(ch.encode_utf8(&mut buffer).as_bytes()); + } + } + fs::write(dir.join("Shell-Core_9701.map"), &bytes).expect("writes"); + + let (registry, outcome) = load_maps_from_dir(&dir).expect("directory reads"); + assert!( + outcome.is_clean(), + "expected a clean load, got {:?}", + outcome.failures + ); + assert_eq!(registry.len(), 1); + } + + #[test] + fn ignores_files_that_are_not_maps() { + let dir = temp_dir("filter"); + write(&dir, "real.map", SHELL_CORE_9701); + write(&dir, "notes.txt", "ignore me"); + write(&dir, "README.md", "ignore me too"); + + let (registry, outcome) = load_maps_from_dir(&dir).expect("directory reads"); + + assert_eq!(registry.len(), 1); + assert!(outcome.is_clean()); + } + + #[test] + fn accepts_an_uppercase_extension() { + let dir = temp_dir("uppercase"); + write(&dir, "real.MAP", SHELL_CORE_9701); + + let (registry, _) = load_maps_from_dir(&dir).expect("directory reads"); + assert_eq!(registry.len(), 1); + } + + #[test] + fn an_empty_directory_loads_cleanly_rather_than_erroring() { + let dir = temp_dir("empty"); + let (registry, outcome) = load_maps_from_dir(&dir).expect("directory reads"); + assert!(registry.is_empty()); + assert!(outcome.is_clean()); + assert_eq!(outcome.loaded_count(), 0); + } + + #[test] + fn a_missing_directory_is_an_error_not_an_empty_result() { + let missing = std::env::temp_dir().join("cmtraceopen-maps-does-not-exist"); + let _ = fs::remove_dir_all(&missing); + assert!(load_maps_from_dir(&missing).is_err()); + } +} + +/// One normalized column produced by a map. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct MappedColumn { + /// Column name, for example `UserName` or `PayloadData1`. + pub property: String, + /// The rendered text. + pub text: String, + /// False when the map referenced a field this event did not carry, in which case `text` still + /// contains the unresolved `%placeholder%`. + pub complete: bool, +} + +fn property_name(property: &MapProperty) -> String { + match property { + MapProperty::UserName => "UserName".to_string(), + MapProperty::RemoteHost => "RemoteHost".to_string(), + MapProperty::ExecutableInfo => "ExecutableInfo".to_string(), + MapProperty::PayloadData(slot) => format!("PayloadData{slot}"), + MapProperty::Other(name) => name.clone(), + // MapProperty is non_exhaustive, so a newer schema can add a target this build predates. + // Naming it after its own debug form keeps the column visible and labelled rather than + // dropping data because the name is unfamiliar. + other => format!("{other:?}"), + } +} + +/// Applies the registered map for this event, if one exists. +/// +/// Returns an empty vector when no map matches, which is the common case: the upstream corpus +/// covers a few hundred event types out of many thousands. +/// +/// The registry is passed in rather than reached for. Both record paths already parse the event +/// once and apply maps there, so whoever owns the registry hands it down; there is no process +/// global to make two tests, or two windows, share one set of maps. +pub fn apply_registered( + registry: &MapRegistry, + channel: &str, + provider: &str, + event_id: u32, + event: &EventNode, +) -> Vec { + let Some(map) = registry.find(channel, provider, event_id) else { + return Vec::new(); + }; + apply_map(map, event) + .values + .into_iter() + .map(|value| MappedColumn { + property: property_name(&value.property), + complete: value.unresolved.is_empty(), + text: value.text, + }) + .collect() +} + +#[cfg(test)] +mod global_tests { + use super::tests::SHELL_CORE_9701; + use super::*; + + #[test] + fn an_unloaded_registry_yields_no_columns_rather_than_failing() { + let event = EventNode::new("Event"); + // Whatever other tests have loaded, an event with no matching map must map to nothing. + assert!(apply_registered( + &MapRegistry::new(), + "No-Such-Channel", + "No-Such-Provider", + 1, + &event + ) + .is_empty()); + } + + #[test] + fn a_loaded_map_produces_columns_for_a_matching_event_end_to_end() { + // Proves the whole chain: YAML on disk, into a registry, applied to XML parsed by the + // host adapter, out as columns the UI can render. The registry is local to this test, so + // it cannot be disturbed by another test loading a different set on a parallel thread. + let dir = std::env::temp_dir().join("cmtraceopen-maps-global-e2e"); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("temp dir"); + fs::write(dir.join("shell-core-9701.map"), SHELL_CORE_9701).expect("write map"); + + let (registry, outcome) = load_maps_from_dir(&dir).expect("loads"); + assert_eq!(outcome.loaded_count(), 1); + assert_eq!(registry.len(), 1); + + let event = crate::event_log::event_node::parse_event_xml( + "RunOnceEx started", + ) + .expect("parses"); + + let columns = apply_registered( + ®istry, + "Microsoft-Windows-Shell-Core/Operational", + "Microsoft-Windows-Shell-Core", + 9701, + &event, + ); + assert_eq!(columns.len(), 1); + assert_eq!(columns[0].property, "PayloadData1"); + assert_eq!(columns[0].text, "RunOnceEx started"); + assert!(columns[0].complete); + + // A different event id on the same channel has no map and must map to nothing. + assert!(apply_registered( + ®istry, + "Microsoft-Windows-Shell-Core/Operational", + "Microsoft-Windows-Shell-Core", + 9702, + &event + ) + .is_empty()); + } + + #[test] + fn property_names_match_the_upstream_column_names() { + assert_eq!(property_name(&MapProperty::UserName), "UserName"); + assert_eq!(property_name(&MapProperty::PayloadData(3)), "PayloadData3"); + assert_eq!( + property_name(&MapProperty::Other("Custom".into())), + "Custom" + ); + } +} diff --git a/src-tauri/src/event_log/mod.rs b/src-tauri/src/event_log/mod.rs index 3962e3dcb..738e5d22e 100644 --- a/src-tauri/src/event_log/mod.rs +++ b/src-tauri/src/event_log/mod.rs @@ -1,6 +1,13 @@ pub mod commands; +pub mod event_node; +pub mod export; +pub mod maps; pub mod models; pub mod parser; +pub mod fetch; +pub mod provider_db; +pub mod rendered; +pub mod timeline; #[cfg(target_os = "windows")] pub mod live; @@ -19,9 +26,48 @@ pub(crate) fn sanitize_control_chars(s: &str) -> String { .to_string() } +/// Parse an event timestamp to epoch milliseconds. +/// +/// Shared by the live and file paths so that a record sorts to the same place regardless of how it +/// was opened. Windows usually writes a full RFC 3339 stamp, but some providers omit the zone, in +/// which case it is read as UTC: the alternative is dropping the event to the epoch, which would +/// silently reorder the timeline rather than being off by a zone offset. +pub(crate) fn parse_timestamp_to_epoch_ms(timestamp: &str) -> i64 { + chrono::DateTime::parse_from_rfc3339(timestamp) + .or_else(|_| { + chrono::NaiveDateTime::parse_from_str(timestamp, "%Y-%m-%dT%H:%M:%S%.f") + .map(|naive| naive.and_utc().fixed_offset()) + }) + .map(|dt| dt.timestamp_millis()) + .unwrap_or(0) +} + #[cfg(test)] mod tests { - use super::sanitize_control_chars; + use super::{parse_timestamp_to_epoch_ms, sanitize_control_chars}; + + #[test] + fn reads_a_full_rfc3339_stamp() { + assert_eq!( + parse_timestamp_to_epoch_ms("2026-08-09T12:00:00.000Z"), + 1_786_276_800_000 + ); + } + + #[test] + fn a_stamp_without_a_zone_is_read_as_utc_rather_than_dropped() { + // Dropping it to zero would sort the event to 1970 and silently reorder the timeline. + assert_eq!( + parse_timestamp_to_epoch_ms("2026-08-09T12:00:00.000"), + parse_timestamp_to_epoch_ms("2026-08-09T12:00:00.000Z") + ); + } + + #[test] + fn an_unreadable_stamp_yields_zero() { + assert_eq!(parse_timestamp_to_epoch_ms("not a time"), 0); + assert_eq!(parse_timestamp_to_epoch_ms(""), 0); + } #[test] fn strips_trailing_carriage_return() { diff --git a/src-tauri/src/event_log/models.rs b/src-tauri/src/event_log/models.rs index 494aca656..911f67ba9 100644 --- a/src-tauri/src/event_log/models.rs +++ b/src-tauri/src/event_log/models.rs @@ -13,9 +13,40 @@ pub struct EvtxRecord { pub level: EvtxLevel, pub computer: String, pub message: String, + #[serde(default)] pub event_data: Vec, + /// The provider's own XML. + /// + /// Defaulted so a caller can omit it. The export command receives records over IPC, and this + /// field dominates the payload: only the XML and JSON formats read it, so sending it for a + /// delimited export serialized every record's XML across the bridge for nothing. + #[serde(default)] pub raw_xml: String, pub source_label: String, + /// Provider-defined task grouping, when the event declares one. + #[serde(default)] + pub task: Option, + /// Operation within the task, when the event declares one. + #[serde(default)] + pub opcode: Option, + /// Emitting process, from `Execution/@ProcessID`. + #[serde(default)] + pub process_id: Option, + /// Emitting thread, from `Execution/@ThreadID`. + #[serde(default)] + pub thread_id: Option, + /// Security identifier from `Security/@UserID`. + /// + /// Kept as the raw SID. Resolving it to an account name needs `LookupAccountSidW` and a cache, + /// and is only meaningful on a machine that knows the domain, so it is a separate concern. + #[serde(default)] + pub user_sid: Option, + /// Keyword bitmask as written by the provider, for example `0x8020000000000000`. + #[serde(default)] + pub keywords: Option, + /// Columns produced by an EvtxECmd map, empty when no map covers this event type. + #[serde(default)] + pub mapped: Vec, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/src-tauri/src/event_log/parser.rs b/src-tauri/src/event_log/parser.rs index e85bf7979..a1bc48412 100644 --- a/src-tauri/src/event_log/parser.rs +++ b/src-tauri/src/event_log/parser.rs @@ -1,27 +1,45 @@ use std::path::Path; +use std::sync::RwLock; +use cmtraceopen_parser::eventmap::MapRegistry; use evtx::EvtxParser; -use serde_json::Value; + +// `extract_event_data` sits in `event_node` alongside `extract_system_fields`: both read a parsed +// tree, and both are needed by the live path as well as this one. Keeping the data extractor here +// while the live path scanned raw XML for itself is what let the two drift apart. +use super::event_node::{extract_event_data, EventFields}; +use super::provider_db::ProviderStore; use super::models::{ ChannelSourceType, EvtxChannelInfo, EvtxField, EvtxLevel, EvtxParseResult, EvtxRecord, }; -use super::sanitize_control_chars; +use super::{parse_timestamp_to_epoch_ms, sanitize_control_chars}; /// Maximum entries to parse from a single .evtx file to prevent memory issues. const MAX_ENTRIES_PER_FILE: usize = 100_000; /// Parse one or more .evtx files and return a unified result. -pub fn parse_evtx_files(paths: &[String]) -> Result { +/// +/// The map registry and provider store are passed in rather than reached for. They belong to the +/// application state, so the caller decides which set is in effect; that is also what lets a test +/// use its own without another test on a parallel thread replacing it. +pub fn parse_evtx_files( + paths: &[String], + maps: &RwLock, + providers: &RwLock, +) -> Result { let mut all_records = Vec::new(); let mut channels = Vec::new(); let mut parse_errors = 0u32; + let mut error_messages = Vec::new(); for path_str in paths { let path = Path::new(path_str); - match parse_single_file(path) { - Ok((records, file_parse_errors)) => { - parse_errors += file_parse_errors; + match parse_single_file(path, maps, providers) { + Ok(file) => { + let records = file.records; + parse_errors += file.parse_errors; + error_messages.extend(file.messages); let source_label = path .file_name() .map(|f| f.to_string_lossy().to_string()) @@ -66,6 +84,9 @@ pub fn parse_evtx_files(paths: &[String]) -> Result { e ); parse_errors += 1; + // A file that could not be opened at all is reported by name. Counting it without + // saying which file, or why, leaves an operator with a number and no next step. + error_messages.push(format!("{path_str}: {e}")); } } } @@ -83,12 +104,31 @@ pub fn parse_evtx_files(paths: &[String]) -> Result { channels, total_records, parse_errors, - error_messages: vec![], + error_messages, }) } -/// Parse a single .evtx file into a Vec of EvtxRecord and a count of per-record parse errors. -fn parse_single_file(path: &Path) -> Result<(Vec, u32), String> { +/// What one file yielded, including why anything was missing from it. +struct ParsedFile { + records: Vec, + /// Records that could not be read. Kept as a count because a damaged file can produce + /// thousands, and thousands of near-identical strings are not worth carrying. + parse_errors: u32, + /// Operator-facing explanations, already summarised. + messages: Vec, +} + +/// Parse a single .evtx file. +/// +/// Anything missing from the result is explained rather than merely counted. A damaged file, a +/// record whose XML will not parse, and a file so large it was truncated are all cases where the +/// view is incomplete, and a view that is silently incomplete is worse than one that is empty: +/// the absent events look like evidence that the thing being investigated did not happen. +fn parse_single_file( + path: &Path, + maps: &RwLock, + providers: &RwLock, +) -> Result { let mut parser = EvtxParser::from_path(path) .map_err(|e| format!("Failed to open EVTX file {}: {}", path.display(), e))?; @@ -99,14 +139,31 @@ fn parse_single_file(path: &Path) -> Result<(Vec, u32), String> { let mut records = Vec::new(); let mut parse_errors = 0u32; - - for record_result in parser.records_json_value() { + let mut messages = Vec::new(); + let mut truncated = false; + + // Locked once for the whole file rather than per record. A hundred thousand lock round trips + // would cost more than the parsing does. + let maps = maps + .read() + .map_err(|_| "map registry lock was poisoned".to_string())?; + // A read guard: looking a provider up caches internally, so it needs no exclusive access. + // Taking the write lock here blocked every other reader for the length of the file. + let providers = providers + .read() + .map_err(|_| "provider store lock was poisoned".to_string())?; + + // XML rather than JSON. The JSON projection cannot be re-parsed into an event tree, which is + // what the map engine, the System block, and the XML export all consume; reading XML here is + // what makes those work on an opened file at all. + for record_result in parser.records() { if records.len() >= MAX_ENTRIES_PER_FILE { log::warn!( "event=evtx_entry_cap_reached file=\"{}\" cap={}", path.display(), MAX_ENTRIES_PER_FILE ); + truncated = true; break; } @@ -123,41 +180,60 @@ fn parse_single_file(path: &Path) -> Result<(Vec, u32), String> { } }; - let json = &record.data; - let system = &json["Event"]["System"]; - let event_data_val = &json["Event"]["EventData"]; - - let provider = system["Provider"]["#attributes"]["Name"] - .as_str() - .unwrap_or("Unknown") - .to_string(); - - let channel = system["Channel"].as_str().unwrap_or("Unknown").to_string(); - - let event_id = extract_event_id(system); - - let level = system["Level"].as_u64().unwrap_or(0) as u8; - let evtx_level = EvtxLevel::from_level_value(level); - - let computer = system["Computer"].as_str().unwrap_or("Unknown").to_string(); - - let timestamp_str = system["TimeCreated"]["#attributes"]["SystemTime"] - .as_str() - .unwrap_or("") - .to_string(); - - let timestamp_epoch = chrono::DateTime::parse_from_rfc3339(×tamp_str) - .map(|dt| dt.timestamp_millis()) - .unwrap_or(0); - + let raw_xml = record.data; let event_record_id = record.event_record_id; - let event_data = extract_event_data(event_data_val); - let message = build_message(&event_data); + // Parsed once and used for identity, the System block, the decoded payload, and any + // registered map, so none of them costs an extra parse. A record whose XML will not parse + // is counted as an error rather than pushed with every field defaulted, which would show a + // row claiming provider "Unknown" at the epoch. + let parsed = match super::event_node::parse_event_xml(&raw_xml) { + Ok(root) => root, + Err(error) => { + log::warn!( + "event=evtx_record_unparsable file=\"{}\" error=\"{}\"", + path.display(), + error + ); + parse_errors += 1; + continue; + } + }; + + let system = super::event_node::extract_system_fields(&parsed); + let provider = system.provider.clone().unwrap_or_else(|| "Unknown".into()); + let channel = system.channel.clone().unwrap_or_else(|| "Unknown".into()); + let event_id = system.event_id.unwrap_or(0); + let evtx_level = EvtxLevel::from_level_value(system.level.unwrap_or(0)); + let computer = system.computer.clone().unwrap_or_else(|| "Unknown".into()); + let timestamp_str = system.time_created.clone().unwrap_or_default(); + let timestamp_epoch = parse_timestamp_to_epoch_ms(×tamp_str); + + let EventFields { + mut fields, + insertions, + } = extract_event_data(&parsed); + + // Same treatment as the live path: a trace-backed event carries its message as hex, and + // without decoding it the row is a wall of digits. + let payload = cmtraceopen_parser::event_payload::decode_payload_in(&parsed) + .map(|decoded| sanitize_control_chars(&decoded.text)); + if let Some(text) = &payload { + // Appended after every real field, so it cannot disturb the positional insertions. + fields.push(EvtxField { + name: "EventPayload".to_string(), + value: text.clone(), + }); + } - // Build raw XML placeholder from JSON (actual XML not available via json_value API) - let raw_xml = serde_json::to_string_pretty(json).unwrap_or_default(); + // A provider database, when one is loaded, turns raw field values into the sentence the + // provider intended. Without it the file path can only summarise EventData, which is what + // every other cross-platform reader shows and why they are hard to read. + let message = describe_event(&providers, &provider, event_id, &insertions) + .or(payload) + .unwrap_or_else(|| super::rendered::build_event_data_summary(&fields)); + let mapped = super::maps::apply_registered(&maps, &channel, &provider, event_id, &parsed); records.push(EvtxRecord { id: 0, // Will be reassigned after sorting event_record_id, @@ -169,66 +245,66 @@ fn parse_single_file(path: &Path) -> Result<(Vec, u32), String> { level: evtx_level, computer, message, - event_data, + event_data: fields, raw_xml, source_label: source_label.clone(), + task: system.task, + opcode: system.opcode, + process_id: system.process_id, + thread_id: system.thread_id, + user_sid: system.user_sid, + keywords: system.keywords, + mapped, }); } - Ok((records, parse_errors)) -} - -/// Extract EventID which can appear as `{"#text": N}` or just `N`. -fn extract_event_id(system: &Value) -> u32 { - if let Some(id) = system["EventID"].as_u64() { - return id as u32; - } - if let Some(id) = system["EventID"]["#text"].as_u64() { - return id as u32; + if truncated { + // Previously only logged. An operator saw exactly the cap as the event count with nothing + // saying the file held more, which reads as a complete picture of a file that was cut off. + messages.push(format!( + "{}: stopped at {} events, the most this reader loads from one file. The file holds more.", + source_label, MAX_ENTRIES_PER_FILE + )); } - if let Some(s) = system["EventID"]["#text"].as_str() { - return s.parse().unwrap_or(0); + if parse_errors > 0 { + messages.push(format!( + "{source_label}: {parse_errors} of {} records could not be read and are missing from the view.", + parse_errors as usize + records.len() + )); } - 0 + + Ok(ParsedFile { + records, + parse_errors, + messages, + }) } -/// Extract EventData fields as key-value pairs. +/// Renders the provider's own description for this event, when metadata for it is loaded. /// -/// All values are sanitized to strip control characters (e.g. `\r`, `\0`) that -/// would render as unexpected glyphs in the UI. -fn extract_event_data(event_data: &Value) -> Vec { - let mut fields = Vec::new(); - - if let Some(obj) = event_data.as_object() { - for (key, value) in obj { - if key == "#attributes" { - continue; - } - let val_str = match value { - Value::String(s) => sanitize_control_chars(s), - Value::Null => continue, - other => sanitize_control_chars(&other.to_string()), - }; - if !val_str.is_empty() { - fields.push(EvtxField { - name: key.clone(), - value: val_str, - }); - } - } +/// Returns `None` when no database is loaded, the provider is absent from it, or the provider does +/// not define this event. Falling back to a field summary is right in all three cases: an absent +/// description is a coverage gap, not a reason to show nothing. +/// +/// A partially rendered description is rejected rather than shown. If the template references +/// insertions the event did not supply, the metadata and the event disagree, and a sentence with +/// `%4` embedded in it is less honest than the field summary it would replace. +fn describe_event( + store: &ProviderStore, + provider: &str, + event_id: u32, + insertions: &[String], +) -> Option { + let metadata = store.provider(provider)?; + let event = metadata.event(event_id, None)?; + let template = event.description.as_deref()?; + + let rendered = cmtraceopen_parser::provider::render_description(template, insertions); + if rendered.is_complete() { + Some(super::sanitize_control_chars(&rendered.text)) + } else { + None } - - fields -} - -/// Build a human-readable message from the first few EventData fields. -fn build_message(event_data: &[EvtxField]) -> String { - event_data - .iter() - .take(5) - .map(|f| format!("{}: {}", f.name, f.value)) - .collect::>() - .join("; ") } #[cfg(test)] @@ -246,51 +322,349 @@ mod tests { assert_eq!(EvtxLevel::from_level_value(255), EvtxLevel::Information); } + /// Empty registries, for tests that only care about parsing. + /// + /// Each test gets its own, so nothing here can be perturbed by another test on a parallel + /// thread loading a different set. + fn empty_state() -> (RwLock, RwLock) { + ( + RwLock::new(MapRegistry::new()), + RwLock::new(ProviderStore::default()), + ) + } + + fn parse(xml: &str) -> cmtraceopen_parser::eventmap::EventNode { + super::super::event_node::parse_event_xml(xml).expect("well formed") + } + + fn fields_of(xml: &str) -> Vec { + extract_event_data(&parse(xml)).fields + } + + fn insertions_of(xml: &str) -> Vec { + extract_event_data(&parse(xml)).insertions + } + #[test] - fn test_extract_event_id_numeric() { - let json: Value = serde_json::json!({"EventID": 4624}); - assert_eq!(extract_event_id(&json), 4624); + fn a_file_that_cannot_be_opened_is_named_in_the_result() { + // A count with no file name and no reason leaves an operator with a number and no next + // step. The message is what makes a missing log actionable. + let (maps, providers) = empty_state(); + let result = parse_evtx_files(&["/no/such/file.evtx".to_string()], &maps, &providers) + .expect("returns"); + assert_eq!(result.parse_errors, 1); + assert_eq!(result.error_messages.len(), 1); + assert!( + result.error_messages[0].contains("/no/such/file.evtx"), + "{:?}", + result.error_messages + ); + assert!(result.records.is_empty()); } #[test] - fn test_extract_event_id_text_object() { - let json: Value = serde_json::json!({"EventID": {"#text": 1001}}); - assert_eq!(extract_event_id(&json), 1001); + fn a_clean_parse_reports_nothing() { + // The messages are a gap report, so an empty run must not manufacture one. + let (maps, providers) = empty_state(); + let result = parse_evtx_files(&[], &maps, &providers).expect("returns"); + assert_eq!(result.parse_errors, 0); + assert!(result.error_messages.is_empty()); + assert_eq!(result.total_records, 0); } #[test] - fn test_extract_event_id_text_string() { - let json: Value = serde_json::json!({"EventID": {"#text": "999"}}); - assert_eq!(extract_event_id(&json), 999); + fn named_event_data_becomes_named_fields() { + let fields = fields_of( + r#" + SYSTEM + 0x3e7 + "#, + ); + assert_eq!(fields.len(), 2); + assert_eq!(fields[0].name, "SubjectUserName"); + assert_eq!(fields[0].value, "SYSTEM"); + assert_eq!(fields[1].name, "TargetLogonId"); } #[test] - fn test_extract_event_data() { - let json: Value = serde_json::json!({ - "#attributes": {"Name": "test"}, - "SubjectUserName": "SYSTEM", - "TargetLogonId": "0x3e7" - }); - let fields = extract_event_data(&json); + fn an_empty_data_element_is_dropped_rather_than_shown_blank() { + let fields = fields_of( + r#" + yes + + "#, + ); + assert_eq!(fields.len(), 1); + assert_eq!(fields[0].name, "Present"); + } + + #[test] + fn unnamed_data_is_numbered_from_one_to_match_insertion_order() { + // Classic providers emit positional Data. Numbering from one lines the fields up with the + // %1 style references in the provider's message template. + let fields = fields_of( + r#" + first + second + "#, + ); + assert_eq!(fields[0].name, "Data1"); + assert_eq!(fields[0].value, "first"); + assert_eq!(fields[1].name, "Data2"); + assert_eq!(fields[1].value, "second"); + } + + #[test] + fn user_data_fields_are_read_through_the_provider_wrapper() { + // Skipping UserData would leave every classic and trace-backed event with no fields at all. + let fields = fields_of( + r#" + + Enforce + C:\app.exe + + "#, + ); + assert_eq!(fields.len(), 2); + assert_eq!(fields[0].name, "PolicyName"); + assert_eq!(fields[1].value, "C:\\app.exe"); + } + + #[test] + fn an_empty_field_still_holds_its_insertion_position() { + // The provider's template addresses fields by position. Dropping the empty one would make + // %3 resolve to what %4 said, and the rendered description would state it as fact. + let xml = r#" + alpha + + gamma + "#; + + assert_eq!(insertions_of(xml), vec!["alpha", "", "gamma"]); + // The display list still omits the blank, because a column of blanks is noise. + assert_eq!( + fields_of(xml) + .iter() + .map(|f| f.name.as_str()) + .collect::>(), + vec!["First", "Third"] + ); + } + + #[test] + fn a_leading_empty_field_does_not_shift_the_rest() { + let xml = "second"; + assert_eq!(insertions_of(xml), vec!["", "second"]); + } + + #[test] + fn a_positional_label_matches_the_slot_the_template_addresses() { + // The label is how an operator matches a field against the provider's template. Skipping + // the count for a blank slot labelled the survivor Data1 while the template calls it %2. + let xml = "second"; + let fields = fields_of(xml); + assert_eq!(fields.len(), 1); + assert_eq!(fields[0].name, "Data2"); + assert_eq!(fields[0].value, "second"); + } + + #[test] + fn insertions_cover_user_data_too() { + let xml = r#" + onethree + "#; + assert_eq!(insertions_of(xml), vec!["one", "", "three"]); + } + + #[test] + fn a_binary_only_event_keeps_its_value() { + // Classic providers emit with no at all. Treating a container without + // as a set of wrappers descends into , finds no children, and drops the + // only value the event carried. + let fields = fields_of("DEADBEEF"); + assert_eq!(fields.len(), 1); + assert_eq!(fields[0].name, "Binary"); + assert_eq!(fields[0].value, "DEADBEEF"); + } + + #[test] + fn data_and_binary_together_both_survive() { + let fields = fields_of( + r#" + timeout + 00FF + "#, + ); assert_eq!(fields.len(), 2); - assert!(fields + assert_eq!(fields[0].name, "Reason"); + assert_eq!(fields[1].name, "Binary"); + } + + #[test] + fn a_wrapper_and_a_direct_field_can_coexist() { + // Decided per child rather than per container, so one shape does not suppress the other. + let fields = fields_of( + r#" + here + there + "#, + ); + assert_eq!(fields.len(), 2); + assert_eq!(fields[0].name, "Direct"); + assert_eq!(fields[1].name, "Nested"); + } + + #[test] + fn positional_numbering_continues_across_containers() { + // The %1 style references in a message template are numbered over the whole event, not + // restarted per container. + let fields = fields_of( + r#" + one + two + "#, + ); + assert_eq!(fields[0].name, "Data1"); + assert_eq!(fields[1].name, "Data2"); + } + + #[test] + fn control_characters_in_a_value_are_stripped() { + let fields = fields_of( + "C:\\app.exe\r", + ); + assert_eq!(fields[0].value, "C:\\app.exe"); + } + + #[test] + fn system_identity_comes_off_the_parsed_tree() { + // The file path used to re-parse a JSON projection as XML, which always failed, leaving + // every System-derived column empty on an opened file. + let system = super::super::event_node::extract_system_fields(&parse( + r#" + + 12 + 2 + + System + TESTHOST-01 + + "#, + )); + assert_eq!( + system.provider.as_deref(), + Some("Microsoft-Windows-Kernel-General") + ); + // The qualifier is a separate value; the id is still the element text. + assert_eq!(system.event_id, Some(12)); + assert_eq!(system.level, Some(2)); + assert_eq!(system.channel.as_deref(), Some("System")); + assert_eq!(system.computer.as_deref(), Some("TESTHOST-01")); + assert_eq!(system.process_id, Some(4)); + assert_eq!( + parse_timestamp_to_epoch_ms(system.time_created.as_deref().unwrap_or_default()), + 1_786_276_800_000 + ); + } +} + +#[cfg(test)] +mod description_tests { + use super::*; + + /// Positional insertions, which is what a description template consumes. + /// + /// Names are kept in the call sites for readability but are not what the template addresses; + /// it refers to fields by position, which is why the insertion list carries empties. + fn insertions(values: &[(&str, &str)]) -> Vec { + values .iter() - .any(|f| f.name == "SubjectUserName" && f.value == "SYSTEM")); + .map(|(_name, value)| value.to_string()) + .collect() + } + + /// A store with nothing registered, which is the state until an operator loads a database. + fn empty_store() -> ProviderStore { + ProviderStore::default() + } + + /// A store with the databases beside `CMTRACEOPEN_PROVIDER_DB` registered. + /// + /// Built per test. A shared one would let these interfere with each other, since registering a + /// directory replaces whatever was there. + fn loaded_store() -> ProviderStore { + let path = std::env::var("CMTRACEOPEN_PROVIDER_DB").expect("database path"); + let directory = std::path::Path::new(&path) + .parent() + .expect("database has a parent directory"); + let mut store = ProviderStore::default(); + store.load_directory(directory).expect("databases load"); + store + } + + #[test] + fn with_no_database_loaded_it_falls_back_to_the_field_summary() { + // The common case until an operator loads metadata. Must not fail or blank the message. + let data = insertions(&[("HRESULT", "0x80180005")]); + assert!(describe_event(&empty_store(), "Nobody-Has-This-Provider", 1, &data).is_none()); + } + + #[test] + #[ignore = "requires a real provider database via CMTRACEOPEN_PROVIDER_DB"] + fn an_unknown_event_id_falls_back_rather_than_inventing_a_description() { + // Needs a store that actually holds the provider. Against an empty one the lookup returns + // None on the provider itself and the event-id branch is never reached, so this only + // repeated the no-database case while claiming to cover something else. + let store = loaded_store(); + let data = insertions(&[("X", "1")]); + assert!( + describe_event( + &store, + "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider", + 999_999, + &data + ) + .is_none(), + "a provider that is loaded but does not define this id must fall back" + ); + } + + #[test] + #[ignore = "requires a real provider database via CMTRACEOPEN_PROVIDER_DB"] + fn a_loaded_database_renders_a_real_provider_description() { + // The whole chain: SQLite on disk, gzip payload, provider metadata, insertion rendering. + let store = loaded_store(); + + let data = insertions(&[("HRESULT", "0x80180005")]); + let described = describe_event( + &store, + "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider", + 2, + &data, + ) + .expect("the MDM provider defines event 2"); + + println!("rendered: {described}"); + assert!(described.contains("0x80180005"), "{described}"); + assert!(!described.contains("%1"), "{described}"); + assert!( + described.len() > "0x80180005".len(), + "a description should be a sentence, not just the value: {described}" + ); } #[test] - fn test_build_message() { - let fields = vec![ - EvtxField { - name: "Key1".into(), - value: "Val1".into(), - }, - EvtxField { - name: "Key2".into(), - value: "Val2".into(), - }, - ]; - let msg = build_message(&fields); - assert_eq!(msg, "Key1: Val1; Key2: Val2"); + #[ignore = "requires a real provider database via CMTRACEOPEN_PROVIDER_DB"] + fn an_event_the_database_does_not_cover_still_falls_back() { + let store = loaded_store(); + + // A provider that genuinely is not in a Windows capture. + assert!(describe_event( + &store, + "Definitely-Not-A-Real-Provider", + 1, + &insertions(&[("a", "b")]) + ) + .is_none()); } } diff --git a/src-tauri/src/event_log/provider_db.rs b/src-tauri/src/event_log/provider_db.rs new file mode 100644 index 000000000..d8051bb79 --- /dev/null +++ b/src-tauri/src/event_log/provider_db.rs @@ -0,0 +1,650 @@ +//! Reading captured provider metadata out of an EventLogExpert provider database. +//! +//! The rendering half lives in `cmtraceopen_parser::provider`, which is pure and works on every +//! platform. This is the host half: it opens the SQLite file, decompresses the payloads, and hands +//! typed metadata across. Keeping SQLite and gzip out of the parser crate is what lets the same +//! rendering run in a wasm build. +//! +//! Format, reverse engineered from a database built on Windows 11 (full spec in issue #539): +//! +//! ```text +//! CREATE TABLE "ProviderDetails" ( +//! "ProviderName" TEXT COLLATE NOCASE, "VersionKey" TEXT, +//! "Events" BLOB, "Keywords" BLOB, "Maps" BLOB, "Messages" BLOB, +//! "Opcodes" BLOB, "Parameters" BLOB, "Tasks" BLOB, +//! "SourceOsBuild" INTEGER, "SourceOsEdition" TEXT, ... +//! PRIMARY KEY ("ProviderName","VersionKey")) +//! ``` +//! +//! Every BLOB is gzip-compressed JSON. A real database holds about 1,180 providers in 16 MB, so +//! rows are read on demand and cached rather than loaded eagerly. + +use std::collections::HashMap; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use cmtraceopen_parser::provider::ProviderMetadata; +use flate2::read::GzDecoder; +use rusqlite::{Connection, OpenFlags}; +use serde::{Deserialize, Serialize}; + +/// What a database contributed, so an operator can see coverage rather than guess at it. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct ProviderDbInfo { + /// Path that was opened. + pub path: String, + /// Number of provider rows. + pub provider_count: u64, + /// Windows build the metadata was captured from, when the rows agree on one. + pub source_os_build: Option, +} + +/// Decompresses one gzip JSON payload into `T`. +/// +/// An empty BLOB is a legitimately empty section rather than a fault, so it deserializes to the +/// type's default instead of erroring. +/// Largest decompressed provider payload accepted from a database. +/// +/// The biggest real provider in a 15.8 MB capture inflates to well under a megabyte, so 64 MB +/// refuses only what could not be a genuine payload. +const MAX_PROVIDER_PAYLOAD_BYTES: u64 = 64 * 1024 * 1024; + +fn inflate_json(blob: &[u8]) -> Result { + if blob.is_empty() { + return Ok(T::default()); + } + // Capped. These databases are evidence supplied by someone else, and an unbounded inflate lets + // a small blob expand to gigabytes and exhaust memory before serde_json ever sees it. The cap + // is far above any real provider payload, so it refuses only what could not be genuine. + let mut decoder = GzDecoder::new(blob).take(MAX_PROVIDER_PAYLOAD_BYTES + 1); + let mut json = String::new(); + decoder + .read_to_string(&mut json) + .map_err(|error| format!("provider payload is not valid gzip: {error}"))?; + if json.len() as u64 > MAX_PROVIDER_PAYLOAD_BYTES { + return Err(format!( + "provider payload inflates past {MAX_PROVIDER_PAYLOAD_BYTES} bytes and was refused" + )); + } + if json.trim().is_empty() { + return Ok(T::default()); + } + serde_json::from_str(&json) + .map_err(|error| format!("provider payload is not valid JSON: {error}")) +} + +/// An open provider database. +/// +/// Debug deliberately reports only the summary; the connection has no useful representation. +pub struct ProviderDb { + connection: Connection, + info: ProviderDbInfo, +} + +impl std::fmt::Debug for ProviderDb { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ProviderDb") + .field("info", &self.info) + .finish_non_exhaustive() + } +} + +impl ProviderDb { + /// Opens `path` read-only. + /// + /// Read-only matters: these databases are evidence supplied by someone else, and opening them + /// writable would let SQLite journal into the directory they arrived in. + pub fn open(path: &Path) -> Result { + let connection = Connection::open_with_flags( + path, + // Read-only, and deliberately without SQLITE_OPEN_URI. With that flag any path + // starting with "file:" is parsed as a URI and its parameters honoured, including + // vfs=. These paths come from scanning a directory the operator pointed at, so a + // file dropped there could otherwise choose how SQLite opens it. + OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .map_err(|error| format!("cannot open provider database {}: {error}", path.display()))?; + + // SQLite integers are signed 64-bit, so rusqlite offers no u64 conversion. Counting rows + // cannot be negative, so widening from i64 is safe and keeps the public type unsigned. + let provider_count: i64 = connection + .query_row("SELECT COUNT(*) FROM ProviderDetails", [], |row| row.get(0)) + .map_err(|error| { + format!( + "{} does not look like a provider database: {error}", + path.display() + ) + })?; + + // Only meaningful when the whole database came from one capture, which is the normal case; + // a merged database reports nothing rather than an arbitrary one of several builds. One + // query answers both halves: MIN and MAX agree exactly when there is a single build. + let source_os_build: Option = connection + .query_row( + "SELECT MIN(SourceOsBuild), MAX(SourceOsBuild) FROM ProviderDetails", + [], + |row| Ok((row.get::<_, Option>(0)?, row.get::<_, Option>(1)?)), + ) + .ok() + .and_then(|(low, high)| match (low, high) { + (Some(low), Some(high)) if low == high => Some(low), + _ => None, + }); + + Ok(Self { + info: ProviderDbInfo { + path: path.display().to_string(), + provider_count: provider_count.max(0) as u64, + source_os_build, + }, + connection, + }) + } + + /// Summary of what this database holds. + pub fn info(&self) -> &ProviderDbInfo { + &self.info + } + + /// Loads one provider's metadata. + /// + /// Provider names are compared case-insensitively, matching the column's `COLLATE NOCASE` and + /// how the event log itself treats them. When several rows exist for one provider, because a + /// database merged captures from different builds, the highest `SourceOsBuild` wins as the + /// closest match to a modern machine. + pub fn provider(&self, name: &str) -> Result, String> { + let mut statement = self + .connection + .prepare_cached( + "SELECT Events, Messages, Tasks, Keywords, Opcodes, SourceOsBuild \ + FROM ProviderDetails WHERE ProviderName = ?1 \ + ORDER BY SourceOsBuild DESC LIMIT 1", + ) + .map_err(|error| format!("cannot prepare provider query: {error}"))?; + + let row = statement.query_row([name], |row| { + Ok(( + row.get::<_, Vec>(0)?, + row.get::<_, Vec>(1)?, + row.get::<_, Vec>(2)?, + row.get::<_, Vec>(3)?, + row.get::<_, Vec>(4)?, + row.get::<_, Option>(5)?, + )) + }); + + let (events, messages, tasks, keywords, opcodes, build) = match row { + Ok(values) => values, + Err(rusqlite::Error::QueryReturnedNoRows) => return Ok(None), + Err(error) => return Err(format!("cannot read provider {name}: {error}")), + }; + + Ok(Some(ProviderMetadata { + provider_name: name.to_string(), + events: inflate_json(&events)?, + messages: inflate_json(&messages)?, + tasks: inflate_json(&tasks)?, + keywords: inflate_json(&keywords)?, + opcodes: inflate_json(&opcodes)?, + source_os_build: build, + })) + } +} + +// ── Store ─────────────────────────────────────────────────────────────────── + +/// Registered provider databases and the metadata read from them. +/// +/// Owned by `AppState` rather than held as a process global. A global made every test share one +/// set: `load_directory` clears and replaces it, and cargo runs tests on parallel threads in one +/// process, so two tests registering databases would interfere. It also meant the registration +/// outlived any workspace the operator closed, with no way to reset it. +#[derive(Default)] +pub struct ProviderStore { + /// Lowercased provider name to metadata, including negative results so a provider absent from + /// every database is not looked up again for every event that mentions it. + /// + /// Behind an `Arc` because a lookup happens once per record. A real provider carries every + /// event it defines with its description strings, so returning it by value meant a deep clone + /// per event: up to a hundred thousand of them to render one file. + /// Behind a `Mutex` so a lookup does not need `&mut self`. Without it the parse path had to + /// hold a write guard on the whole store for the length of a file, blocking every other reader + /// just to populate a cache. + cache: Mutex>>>, + /// The registered databases, opened once at registration and reused for every lookup. + open_databases: Mutex>, + info: Vec, +} + +impl ProviderStore { + /// Registers every `.db` in `directory`, replacing any previously registered set. + pub fn load_directory(&mut self, directory: &Path) -> Result, String> { + let entries = std::fs::read_dir(directory).map_err(|error| { + format!( + "cannot read provider database directory {}: {error}", + directory.display() + ) + })?; + + let mut databases: Vec = Vec::new(); + let mut info = Vec::new(); + let mut failures: Vec = Vec::new(); + + let mut paths: Vec = Vec::new(); + for entry in entries { + // An enumeration error after the directory opened is recorded, not skipped, matching + // the map loader. Dropping it lets a partial set look complete. + match entry { + Ok(entry) => paths.push(entry.path()), + Err(error) => failures.push(format!( + "cannot read an entry in {}: {error}", + directory.display() + )), + } + } + paths.retain(|path| { + path.is_file() + && path + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("db")) + }); + paths.sort(); + + for path in paths { + match ProviderDb::open(&path) { + Ok(database) => { + info.push(database.info().clone()); + // Kept open. Reopening per lookup also re-ran the schema probe inside open(), + // once per registered database for every distinct provider name in a file. + databases.push(database); + } + // A file that is not a provider database is reported, not fatal: the directory is + // user-supplied and may hold anything. + Err(reason) => failures.push(reason), + } + } + + // The locks are taken before anything is published, and a failure aborts rather than + // continuing. This takes &mut self, so neither lock can be contended and the only failure + // is poisoning; swallowing it left `info` describing databases that were never opened, so + // registered() reported coverage that no lookup could deliver. + let mut open = self + .open_databases + .lock() + .map_err(|_| "provider store lock was poisoned".to_string())?; + let mut cache = self + .cache + .lock() + .map_err(|_| "provider cache lock was poisoned".to_string())?; + *open = databases; + cache.clear(); + drop(open); + drop(cache); + self.info = info.clone(); + + if info.is_empty() && !failures.is_empty() { + return Err(failures.join("; ")); + } + // Reported even when something loaded. Returning Ok and dropping four reasons on the floor + // leaves an operator looking at partial provider coverage with no explanation for it. + for failure in &failures { + log::warn!( + "event=provider_db_skipped directory=\"{}\" reason=\"{failure}\"", + directory.display() + ); + } + Ok(info) + } + + /// Metadata for `provider_name`, consulting registered databases in order and caching it. + /// + /// The cache is behind its own lock, so this needs only `&self`; a lookup still populates it, + /// including the negative result, which is what stops a provider absent from every database + /// being searched again for every event that names it. + pub fn provider(&self, provider_name: &str) -> Option> { + let key = provider_name.to_ascii_lowercase(); + if let Ok(cache) = self.cache.lock() { + if let Some(cached) = cache.get(&key) { + return cached.clone(); + } + } + + // Databases are opened once and held, not reopened per lookup. Opening also runs a schema + // probe, so a miss used to pay an open plus that probe for every registered database, for + // every distinct provider name in the file. + let mut found = None; + if let Ok(open) = self.open_databases.lock() { + for database in open.iter() { + if let Ok(Some(metadata)) = database.provider(provider_name) { + found = Some(metadata); + break; + } + } + } + + let found = found.map(Arc::new); + if let Ok(mut cache) = self.cache.lock() { + cache.insert(key, found.clone()); + } + found + } + + /// Summary of every registered database. + pub fn registered(&self) -> Vec { + self.info.clone() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use flate2::write::GzEncoder; + use flate2::Compression; + use std::io::Write; + + fn gzip(json: &str) -> Vec { + let mut encoder = GzEncoder::new(Vec::new(), Compression::fast()); + encoder.write_all(json.as_bytes()).expect("compress"); + encoder.finish().expect("finish") + } + + /// Builds a database with the schema observed on Windows 11. + fn build_db(path: &Path, providers: &[(&str, u32, &str)]) { + let connection = Connection::open(path).expect("create"); + connection + .execute_batch( + r#"CREATE TABLE "ProviderDetails" ( + "ProviderName" TEXT COLLATE NOCASE NOT NULL, + "VersionKey" TEXT NOT NULL, + "Events" BLOB NOT NULL, "Keywords" BLOB NOT NULL, "Maps" BLOB NOT NULL, + "Messages" BLOB NOT NULL, "Opcodes" BLOB NOT NULL, "Parameters" BLOB NOT NULL, + "Tasks" BLOB NOT NULL, "SourceOsBuild" INTEGER, + PRIMARY KEY ("ProviderName","VersionKey"));"#, + ) + .expect("schema"); + + for (name, build, events_json) in providers { + connection + .execute( + r#"INSERT INTO ProviderDetails + (ProviderName, VersionKey, Events, Keywords, Maps, Messages, Opcodes, + Parameters, Tasks, SourceOsBuild) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"#, + rusqlite::params![ + name, + format!("vk1:{build}"), + gzip(events_json), + gzip(r#"{"1":"Error"}"#), + gzip("{}"), + gzip("[]"), + gzip(r#"{"11":"Start"}"#), + gzip("[]"), + gzip(r#"{"1":"Enrollment"}"#), + build, + ], + ) + .expect("insert"); + } + } + + const EVENTS: &str = r#"[{"Id":2,"Version":0,"Description":"Enroll failed: (%1).","Level":2}]"#; + + fn temp_dir(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("cmtraceopen-providerdb-{name}")); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("temp dir"); + dir + } + + #[test] + fn reads_a_provider_and_inflates_its_payloads() { + let dir = temp_dir("read"); + let path = dir.join("base.db"); + build_db(&path, &[("Test-Provider", 26200, EVENTS)]); + + let database = ProviderDb::open(&path).expect("opens"); + assert_eq!(database.info().provider_count, 1); + assert_eq!(database.info().source_os_build, Some(26200)); + + let metadata = database + .provider("Test-Provider") + .expect("query") + .expect("provider present"); + assert_eq!(metadata.events.len(), 1); + assert_eq!(metadata.events[0].id, 2); + assert_eq!(metadata.task_name(1), Some("Enrollment")); + assert_eq!(metadata.opcode_name(11), Some("Start")); + assert_eq!(metadata.keyword_names(1), vec!["Error"]); + } + + #[test] + fn provider_lookup_is_case_insensitive_like_the_column_and_the_event_log() { + let dir = temp_dir("case"); + let path = dir.join("base.db"); + build_db(&path, &[("Test-Provider", 26200, EVENTS)]); + + let database = ProviderDb::open(&path).expect("opens"); + assert!(database.provider("test-PROVIDER").expect("query").is_some()); + } + + #[test] + fn an_unknown_provider_is_absent_rather_than_an_error() { + let dir = temp_dir("absent"); + let path = dir.join("base.db"); + build_db(&path, &[("Test-Provider", 26200, EVENTS)]); + + let database = ProviderDb::open(&path).expect("opens"); + assert!(database.provider("Nobody").expect("query").is_none()); + } + + #[test] + fn the_newest_capture_wins_when_a_provider_appears_more_than_once() { + let dir = temp_dir("versions"); + let path = dir.join("base.db"); + build_db( + &path, + &[ + ( + "Dup", + 22000, + r#"[{"Id":1,"Version":0,"Description":"old"}]"#, + ), + ( + "Dup", + 26200, + r#"[{"Id":1,"Version":0,"Description":"new"}]"#, + ), + ], + ); + + let metadata = ProviderDb::open(&path) + .expect("opens") + .provider("Dup") + .expect("query") + .expect("present"); + assert_eq!( + metadata.events[0].description.as_deref(), + Some("new"), + "the newest captured build should win" + ); + } + + #[test] + fn a_merged_database_reports_no_single_source_build() { + let dir = temp_dir("merged"); + let path = dir.join("base.db"); + build_db(&path, &[("A", 22000, EVENTS), ("B", 26200, EVENTS)]); + + // Reporting one of several builds would misrepresent where the metadata came from. + assert_eq!( + ProviderDb::open(&path) + .expect("opens") + .info() + .source_os_build, + None + ); + } + + #[test] + fn a_file_that_is_not_a_provider_database_is_rejected_clearly() { + let dir = temp_dir("notadb"); + let path = dir.join("junk.db"); + std::fs::write(&path, b"this is not sqlite").expect("write"); + + let error = ProviderDb::open(&path).expect_err("should fail"); + assert!( + error.contains("does not look like a provider database"), + "{error}" + ); + } + + #[test] + fn an_empty_payload_is_an_empty_section_not_a_fault() { + let empty: Vec = Vec::new(); + let events: Vec = + inflate_json(&empty).expect("empty blob is fine"); + assert!(events.is_empty()); + } + + #[test] + fn a_corrupt_payload_is_reported_rather_than_silently_empty() { + let result: Result, String> = + inflate_json(b"not gzip at all"); + assert!(result.is_err(), "corrupt payloads must not read as empty"); + } + + #[test] + fn loading_a_directory_registers_every_database_and_skips_other_files() { + let dir = temp_dir("directory"); + build_db(&dir.join("a.db"), &[("A", 26200, EVENTS)]); + build_db(&dir.join("b.db"), &[("B", 26200, EVENTS)]); + std::fs::write(dir.join("notes.txt"), b"ignore me").expect("write"); + + // A store local to this test. The old process global meant a parallel test registering a + // different directory replaced this one's set mid-run. + let mut store = ProviderStore::default(); + let info = store.load_directory(&dir).expect("loads"); + assert_eq!(info.len(), 2); + assert!(store.provider("A").is_some()); + assert!(store.provider("B").is_some()); + assert!(store.provider("Nobody").is_none()); + assert_eq!(store.registered().len(), 2); + } +} + +#[cfg(test)] +mod real_database_tests { + //! Exercises the reader against a provider database produced by EventLogExpert's own tool. + //! + //! Ignored by default: it needs a real `.db`, whose path comes from + //! `CMTRACEOPEN_PROVIDER_DB`. Synthetic fixtures prove the reader handles the schema as I + //! understand it; only a real file proves I understood it. + //! + //! ```text + //! CMTRACEOPEN_PROVIDER_DB=C:\path\to.db \ + //! cargo test --lib event_log::provider_db::real_database_tests -- --ignored --nocapture + //! ``` + + use super::*; + + fn real_db() -> Option { + let path = std::env::var("CMTRACEOPEN_PROVIDER_DB").ok()?; + ProviderDb::open(Path::new(&path)).ok() + } + + #[test] + #[ignore = "requires a real provider database via CMTRACEOPEN_PROVIDER_DB"] + fn opens_a_real_database_and_reports_its_size() { + let database = real_db().expect("database opens"); + let info = database.info(); + println!( + "providers={} source_os_build={:?}", + info.provider_count, info.source_os_build + ); + assert!( + info.provider_count > 100, + "a machine-wide capture should hold hundreds of providers, got {}", + info.provider_count + ); + } + + #[test] + #[ignore = "requires a real provider database via CMTRACEOPEN_PROVIDER_DB"] + fn renders_a_real_mdm_description_end_to_end() { + use cmtraceopen_parser::provider::render_description; + + let database = real_db().expect("database opens"); + let metadata = database + .provider("Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider") + .expect("query succeeds") + .expect("the MDM provider is present on a Windows machine"); + + println!( + "MDM provider: {} events, {} tasks, {} keywords", + metadata.events.len(), + metadata.tasks.len(), + metadata.keywords.len() + ); + assert!( + metadata.events.len() > 50, + "the MDM provider defines many events, got {}", + metadata.events.len() + ); + + let event = metadata.event(2, Some(0)).expect("event id 2 is defined"); + let template = event + .description + .as_deref() + .expect("event 2 has a description"); + println!("template: {template}"); + + let rendered = render_description(template, &["0x80180005".to_string()]); + println!("rendered: {}", rendered.text); + assert!( + !rendered.text.contains("%1"), + "the insertion should have been filled: {}", + rendered.text + ); + assert!(rendered.is_complete()); + } + + #[test] + #[ignore = "requires a real provider database via CMTRACEOPEN_PROVIDER_DB"] + fn every_payload_in_a_sample_of_providers_inflates() { + // A decompression or schema misunderstanding would show up here rather than as one + // provider quietly rendering nothing. + let database = real_db().expect("database opens"); + let names: Vec = { + let mut statement = database + .connection + .prepare("SELECT ProviderName FROM ProviderDetails LIMIT 200") + .expect("prepare"); + let rows = statement + .query_map([], |row| row.get::<_, String>(0)) + .expect("query"); + rows.filter_map(Result::ok).collect() + }; + assert!(!names.is_empty()); + + let mut with_events = 0usize; + for name in &names { + let metadata = database + .provider(name) + .unwrap_or_else(|error| panic!("provider {name} failed to inflate: {error}")) + .unwrap_or_else(|| panic!("provider {name} vanished between listing and reading")); + if !metadata.events.is_empty() { + with_events += 1; + } + } + println!( + "{}/{} sampled providers define events", + with_events, + names.len() + ); + assert!(with_events > 0); + } +} diff --git a/src-tauri/src/event_log/rendered.rs b/src-tauri/src/event_log/rendered.rs new file mode 100644 index 000000000..7c8e5efb6 --- /dev/null +++ b/src-tauri/src/event_log/rendered.rs @@ -0,0 +1,363 @@ +//! Turning one rendered event into one [`EvtxRecord`]. +//! +//! The live path renders each event to XML with `EvtRender` and then has to translate it. That +//! translation is pure: it takes a string and returns a record, and nothing about it needs the +//! Event Log service. It lives here rather than in `live` so it compiles and is tested on every +//! platform, not only the one that can produce the input. +//! +//! It reads the parsed tree that the map engine already needs, rather than scanning the XML text. +//! The scanning version this replaces re-derived six System fields with substring searches and +//! matched `EventData` with a regular expression, on top of the parse it was doing anyway. That +//! regex required a `Name` attribute, could not match a value containing a newline, and never saw +//! `UserData` at all, so three whole classes of event field were dropped from the live view without +//! anything indicating a field was missing. The file path never had those bugs because it read the +//! tree. Sharing one extractor is what stops the two paths drifting again. + +use cmtraceopen_parser::eventmap::{EventNode, MapRegistry}; + +use super::event_node::{ + extract_event_data, extract_system_fields, parse_event_xml, EventFields, SystemFields, +}; +use super::models::{EvtxField, EvtxLevel, EvtxRecord}; +use super::{parse_timestamp_to_epoch_ms, sanitize_control_chars}; + +/// Names a provider that identified itself in neither of the two ways it can. +/// +/// Matches what the file path shows for the same event. A blank cell reads as a missing column +/// rather than as an event that did not name its source. +const UNKNOWN: &str = "Unknown"; + +/// Builds a record from one rendered event. +/// +/// Returns `None` when the XML will not parse. The caller counts those rather than pushing a record +/// with every field defaulted, which would put a row on screen claiming provider "Unknown" at the +/// epoch and looking exactly like a real event that happened in 1970. +/// +/// The live path calls [`record_from_parts`] instead, because it needs the provider name before the +/// record exists in order to ask the service to render the message. +pub fn parse_xml_to_record( + xml: &str, + channel: &str, + maps: &MapRegistry, + rendered_message: Option<&str>, +) -> Option { + let parsed = parse_event_xml(xml).ok()?; + let system = extract_system_fields(&parsed); + Some(record_from_parts( + &parsed, + system, + xml, + channel, + maps, + rendered_message, + )) +} + +/// Builds a record from a tree and System block the caller has already read. +/// +/// Both are taken as arguments so the document is parsed once per event. The live path needs the +/// provider name first, to name the publisher whose message template it asks the service to render, +/// and parsing again to get the rest would double the cost of the hottest loop in the view. +/// +/// `parsed` is used for the System block, the event fields, the decoded payload and any registered +/// map, so none of them costs an extra pass over the document. +pub fn record_from_parts( + parsed: &EventNode, + system: SystemFields, + xml: &str, + channel: &str, + maps: &MapRegistry, + rendered_message: Option<&str>, +) -> EvtxRecord { + // `Provider` carries `Name` for a manifest provider and `EventSourceName` for a classic one. + // Some carry both, and then `Name` is the identity. `extract_system_fields` already encodes + // that; the live path used to search the raw text for `Name=`, which also matches the tail of + // `EventSourceName=`. + let provider = system.provider.unwrap_or_else(|| UNKNOWN.to_string()); + let event_id = system.event_id.unwrap_or(0); + // Defaulted to the same value the file path uses, so one event cannot show two severities + // depending on how it was opened. `EvtxLevel` maps every unrecognised value to Information, so + // this is Information in practice; it is written as 0 to keep the two paths textually identical + // if that mapping ever gains a distinct "unspecified" variant. + let level = EvtxLevel::from_level_value(system.level.unwrap_or(0)); + let computer = system.computer.unwrap_or_else(|| UNKNOWN.to_string()); + let timestamp = system.time_created.unwrap_or_default(); + let timestamp_epoch = parse_timestamp_to_epoch_ms(×tamp); + let event_record_id = system.event_record_id.unwrap_or(0); + + // The live path renders its message with `EvtFormatMessage`, which does its own substitution + // inside the service, so the positional insertion list has no consumer here. + let EventFields { + fields: mut event_data, + insertions: _, + } = extract_event_data(parsed); + + // Trace-backed channels carry their message as a hex blob rather than as EventData, so without + // this the row reads as a wall of hex digits. Surfaced as a field of its own because the raw + // hex never appears in EventData. Appended after every real field so it cannot disturb the + // positional insertions. + let payload = cmtraceopen_parser::event_payload::decode_payload_in(parsed) + .map(|decoded| sanitize_control_chars(&decoded.text)); + if let Some(text) = &payload { + event_data.push(EvtxField { + name: "EventPayload".to_string(), + value: text.clone(), + }); + } + + // The provider's own rendered message wins when there is one. A decoded payload comes next, + // used whole rather than through the summary, which would truncate the only text the event has + // at 80 characters. Sanitized to strip control characters that render as unexpected glyphs. + let message = rendered_message + .map(sanitize_control_chars) + .or(payload) + .unwrap_or_else(|| build_event_data_summary(&event_data)); + + let mapped = super::maps::apply_registered(maps, channel, &provider, event_id, parsed); + + EvtxRecord { + id: 0, // assigned by commands.rs after sorting + event_record_id, + timestamp, + timestamp_epoch, + provider, + // The queried channel names the record. The caller knows which channel it asked for, and a + // forwarded event names the channel it came from rather than the one holding it. + channel: channel.to_string(), + event_id, + level, + computer, + message, + event_data, + raw_xml: xml.to_string(), + source_label: "Live".to_string(), + task: system.task, + opcode: system.opcode, + process_id: system.process_id, + thread_id: system.thread_id, + user_sid: system.user_sid, + keywords: system.keywords, + mapped, + } +} + +/// Builds a summary from event fields, for events the provider will not describe. +pub fn build_event_data_summary(fields: &[EvtxField]) -> String { + fields + .iter() + .take(5) + .map(|f| { + let val = if f.value.chars().count() > 80 { + // Sliced by character rather than by byte. A value whose 78th byte lands inside a + // multi-byte character panics on a byte slice, and event fields carry paths and + // user names that are routinely not ASCII. + let head: String = f.value.chars().take(77).collect(); + format!("{head}...") + } else { + f.value.clone() + }; + format!("{}: {val}", f.name) + }) + .collect::>() + .join("; ") +} + +#[cfg(test)] +mod tests { + //! `EvtRender` writes attributes in single quotes, so the fixtures do too. Double-quoted + //! fixtures would exercise a shape the service never emits. + + use super::*; + + fn record_for(xml: &str, channel: &str) -> EvtxRecord { + parse_xml_to_record(xml, channel, &MapRegistry::new(), None) + .expect("a well formed event should produce a record") + } + + fn field<'a>(record: &'a EvtxRecord, name: &str) -> Option<&'a str> { + record + .event_data + .iter() + .find(|f| f.name == name) + .map(|f| f.value.as_str()) + } + + /// Manifest provider: `Name` is present, and that is the identity to use. + const MANIFEST: &str = r#" + + + 4624 + 0 + + 4242 + Security + RING0IVY24-01 + + + 0x8020000000000000 + + + adam + +"#; + + #[test] + fn a_manifest_provider_is_read_from_name() { + let record = record_for(MANIFEST, "Security"); + assert_eq!(record.provider, "Microsoft-Windows-Security-Auditing"); + assert_eq!(record.event_id, 4624); + assert_eq!(record.event_record_id, 4242); + assert_eq!(record.computer, "RING0IVY24-01"); + assert_eq!(record.process_id, Some(1234)); + assert_eq!(record.thread_id, Some(5678)); + assert_eq!(record.user_sid.as_deref(), Some("S-1-5-18")); + assert_eq!(record.keywords.as_deref(), Some("0x8020000000000000")); + assert_eq!(field(&record, "TargetUserName"), Some("adam")); + } + + #[test] + fn a_classic_source_still_names_its_provider() { + // A classic source names itself only in EventSourceName. Losing it leaves the row blank and + // stops every map from matching, and neither failure looks like a failure. + let xml = r#" + + + 10 + 4 + Application + +"#; + let record = record_for(xml, "Application"); + assert_eq!(record.provider, "Print"); + assert_eq!(record.event_id, 10, "Qualifiers is not part of the id"); + } + + #[test] + fn a_manifest_name_wins_over_a_legacy_source_alias() { + // Both attributes are present, and the text `Name='` occurs inside `EventSourceName='`. + // Anything searching the raw XML for that substring can match the alias and report the + // wrong provider depending only on attribute order. + let xml = r#" + + + 16384 + +"#; + assert_eq!( + record_for(xml, "Application").provider, + "Microsoft-Windows-Security-SPP" + ); + } + + #[test] + fn an_unnamed_data_element_is_kept_and_numbered_by_position() { + // Classic providers write positional Data with no Name. Requiring a Name attribute dropped + // every one of them, so the event arrived with no fields and the detail pane was empty. + let xml = r#" + 1033 + + Product Name + 1.2.3 + +"#; + let record = record_for(xml, "Application"); + assert_eq!(field(&record, "Data1"), Some("Product Name")); + assert_eq!(field(&record, "Data2"), Some("1.2.3")); + } + + #[test] + fn a_multi_line_value_survives() { + // Stack traces, path lists and MSI output all span lines. A regex whose `.` stops at a + // newline matched none of them, and the field vanished rather than arriving truncated, + // which is the harder failure to notice. + let xml = "\n 1000\n \n at Frame.One\nat Frame.Two\n \n"; + assert_eq!( + field(&record_for(xml, "Application"), "Trace"), + Some("at Frame.One\nat Frame.Two") + ); + } + + #[test] + fn user_data_fields_are_read() { + // Trace-backed and classic providers put their fields in UserData, nested under a + // provider-named wrapper. Reading only EventData left these events with no fields at all. + let xml = r#" + 811 + + + WinlogonSubscription + + +"#; + assert_eq!( + field(&record_for(xml, "Application"), "SubscriptionId"), + Some("WinlogonSubscription") + ); + } + + #[test] + fn an_absent_level_defaults_to_what_the_file_path_shows() { + // The two paths must default identically, or the same event shows one severity when opened + // from a file and another when read live. They previously wrote different literals, 0 here + // and 4 there, which happens to be the same value only because `EvtxLevel` folds every + // unrecognised level into Information. + let xml = r#" + 1 +"#; + assert_eq!( + record_for(xml, "Application").level, + EvtxLevel::from_level_value(0) + ); + } + + #[test] + fn an_unnamed_provider_is_reported_as_unknown_not_as_blank() { + let xml = r#" + 1 +"#; + assert_eq!(record_for(xml, "Application").provider, UNKNOWN); + } + + #[test] + fn malformed_xml_yields_no_record_rather_than_an_empty_one() { + // A record with every field defaulted is indistinguishable on screen from a real event that + // happened at the epoch with no provider. + assert!( + parse_xml_to_record("", "Application", &MapRegistry::new(), None) + .is_none() + ); + } + + #[test] + fn the_rendered_message_is_preferred_over_a_field_summary() { + let record = parse_xml_to_record( + MANIFEST, + "Security", + &MapRegistry::new(), + Some("An account was successfully logged on."), + ) + .expect("record"); + assert_eq!(record.message, "An account was successfully logged on."); + } + + #[test] + fn without_a_rendered_message_the_fields_are_summarised() { + assert_eq!( + record_for(MANIFEST, "Security").message, + "TargetUserName: adam" + ); + } + + #[test] + fn a_long_non_ascii_value_is_truncated_without_panicking() { + // Truncating by byte offset panics when the cut lands inside a multi-byte character, and + // event fields carry paths and account names that are routinely not ASCII. + let value = "é".repeat(200); + let summary = build_event_data_summary(&[EvtxField { + name: "Path".to_string(), + value, + }]); + assert!(summary.ends_with("...")); + assert_eq!(summary.chars().filter(|c| *c == 'é').count(), 77); + } +} diff --git a/src-tauri/src/event_log/timeline.rs b/src-tauri/src/event_log/timeline.rs new file mode 100644 index 000000000..36d2c4f75 --- /dev/null +++ b/src-tauri/src/event_log/timeline.rs @@ -0,0 +1,219 @@ +//! Placing Windows events onto the unified timeline. +//! +//! The merge itself lives in `cmtraceopen_parser::unified_timeline`, which is pure and knows +//! nothing about where items came from. This is the event-side adapter: it converts an +//! [`EvtxRecord`] into a timeline item, or reports why it cannot be placed. +//! +//! The log side needs no adapter because `LogEntry` already lives in the parser crate. + +use cmtraceopen_parser::models::log_entry::LogEntry; +use cmtraceopen_parser::unified_timeline::{ + from_log_entry, merge, TimelineItem, TimelineOrigin, TimelineSeverity, UnifiedTimeline, + UnplacedItem, UnplacedReason, +}; + +use super::models::{EvtxLevel, EvtxRecord}; + +/// Maps the record's level back to a timeline severity. +/// +/// `EvtxRecord` stores a decoded level rather than the raw `System/Level` value, so this maps the +/// decoded form. Information is the resting state, matching how the decoder treats a level it does +/// not recognise. +fn severity_of(level: EvtxLevel) -> TimelineSeverity { + match level { + EvtxLevel::Critical => TimelineSeverity::Critical, + EvtxLevel::Error => TimelineSeverity::Error, + EvtxLevel::Warning => TimelineSeverity::Warning, + EvtxLevel::Verbose => TimelineSeverity::Verbose, + EvtxLevel::Information => TimelineSeverity::Info, + } +} + +fn origin_of(record: &EvtxRecord) -> TimelineOrigin { + TimelineOrigin::Event { + channel: record.channel.clone(), + provider: record.provider.clone(), + event_id: record.event_id, + record_id: record.event_record_id, + } +} + +/// Converts one event, or reports why it has no position. +/// +/// A record whose timestamp did not parse carries `timestamp_epoch == 0`, which is 1970 and not a +/// time any Windows event was written. Treating it as a real position would drop the event at the +/// far left of every timeline and imply it happened first. +pub fn from_event(record: &EvtxRecord) -> Result { + if record.timestamp_epoch == 0 { + return Err(UnplacedItem { + origin: origin_of(record), + reason: UnplacedReason::MissingTimestamp, + }); + } + + Ok(TimelineItem { + timestamp_ms: record.timestamp_epoch, + severity: severity_of(record.level), + message: record.message.clone(), + origin: origin_of(record), + }) +} + +/// Builds one timeline from parsed log entries and events. +pub fn build(entries: &[LogEntry], records: &[EvtxRecord]) -> UnifiedTimeline { + let mut placed = Vec::with_capacity(entries.len() + records.len()); + let mut unplaced = Vec::new(); + + for entry in entries { + match from_log_entry(entry) { + Ok(item) => placed.push(item), + Err(reason) => unplaced.push(reason), + } + } + for record in records { + match from_event(record) { + Ok(item) => placed.push(item), + Err(reason) => unplaced.push(reason), + } + } + + merge(placed, unplaced) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn record(timestamp_epoch: i64, message: &str, level: EvtxLevel) -> EvtxRecord { + EvtxRecord { + id: 0, + // Distinct from event_id below, so a test cannot pass while reading the wrong one. + event_record_id: 1234, + timestamp: String::new(), + timestamp_epoch, + provider: "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider" + .to_string(), + channel: "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider/Admin" + .to_string(), + event_id: 76, + level, + computer: "TESTHOST-01".to_string(), + message: message.to_string(), + event_data: Vec::new(), + raw_xml: String::new(), + source_label: "Live".to_string(), + task: None, + opcode: None, + process_id: None, + thread_id: None, + user_sid: None, + keywords: None, + mapped: Vec::new(), + } + } + + fn entry(timestamp: Option, message: &str) -> LogEntry { + LogEntry { + line_number: 12, + message: message.to_string(), + component: Some("IME".to_string()), + timestamp, + file_path: "C:/logs/IntuneManagementExtension.log".to_string(), + ..LogEntry::default() + } + } + + #[test] + fn an_event_and_the_log_lines_around_it_interleave() { + // The case the whole feature exists for. + let timeline = build( + &[ + entry(Some(1_000), "Checking enrollment"), + entry(Some(3_000), "Token request rejected"), + ], + &[record(2_000, "MDM enroll failed", EvtxLevel::Error)], + ); + + let messages: Vec<&str> = timeline.items.iter().map(|i| i.message.as_str()).collect(); + assert_eq!( + messages, + vec![ + "Checking enrollment", + "MDM enroll failed", + "Token request rejected" + ] + ); + assert!(timeline.is_complete()); + } + + #[test] + fn an_event_with_no_parsed_timestamp_is_unplaced_rather_than_dated_to_1970() { + // Epoch zero would put it at the far left of every timeline and imply it happened first. + let timeline = build(&[], &[record(0, "undated", EvtxLevel::Error)]); + assert!(timeline.items.is_empty()); + assert_eq!(timeline.unplaced.len(), 1); + assert_eq!( + timeline.unplaced[0].reason, + UnplacedReason::MissingTimestamp + ); + } + + #[test] + fn an_unplaced_event_still_identifies_itself() { + let timeline = build(&[], &[record(0, "undated", EvtxLevel::Error)]); + match &timeline.unplaced[0].origin { + TimelineOrigin::Event { + channel, + event_id, + record_id, + .. + } => { + assert!(channel.contains("DeviceManagement")); + // Distinct values, so mapping event_id to record_id or the reverse fails here. + assert_eq!(*event_id, 76); + assert_eq!(*record_id, 1234); + } + other => panic!("expected an event origin, got {other:?}"), + } + } + + #[test] + fn unplaced_items_from_both_sides_are_collected_together() { + let timeline = build( + &[entry(None, "continuation line")], + &[record(0, "undated event", EvtxLevel::Information)], + ); + assert_eq!(timeline.unplaced.len(), 2); + assert!(!timeline.is_complete()); + } + + #[test] + fn every_level_maps_without_panicking() { + // Every arm asserted, not just the two ends. The loop previously only checked the + // timestamp, which is independent of the level, so a wrong arm for Warning, Error or + // Information passed. + for (level, expected) in [ + (EvtxLevel::Critical, TimelineSeverity::Critical), + (EvtxLevel::Error, TimelineSeverity::Error), + (EvtxLevel::Warning, TimelineSeverity::Warning), + (EvtxLevel::Information, TimelineSeverity::Info), + (EvtxLevel::Verbose, TimelineSeverity::Verbose), + ] { + let item = from_event(&record(1, "x", level)).expect("placed"); + assert_eq!(item.timestamp_ms, 1); + assert_eq!( + item.severity, expected, + "{level:?} must map to {expected:?}" + ); + assert_eq!(severity_of(level), expected); + } + } + + #[test] + fn an_empty_build_is_complete_and_empty() { + let timeline = build(&[], &[]); + assert!(timeline.items.is_empty()); + assert!(timeline.is_complete()); + assert_eq!(timeline.span_ms(), None); + } +} diff --git a/src-tauri/src/intune/evtx_parser.rs b/src-tauri/src/intune/evtx_parser.rs index b02a38597..5c3e6915e 100644 --- a/src-tauri/src/intune/evtx_parser.rs +++ b/src-tauri/src/intune/evtx_parser.rs @@ -617,8 +617,7 @@ fn has_valid_xml_declaration(declaration: &BytesDecl<'_>, decoder: Decoder) -> b let Ok(attribute) = attribute else { return false; }; - let Ok(value) = - attribute.decoded_and_normalized_value(XmlVersion::Implicit1_0, decoder) + let Ok(value) = attribute.decoded_and_normalized_value(XmlVersion::Implicit1_0, decoder) else { return false; }; @@ -672,8 +671,7 @@ fn has_valid_xml_attributes(start: &BytesStart<'_>, decoder: Decoder) -> bool { let Ok(attribute) = attribute else { return false; }; - let Ok(value) = - attribute.decoded_and_normalized_value(XmlVersion::Implicit1_0, decoder) + let Ok(value) = attribute.decoded_and_normalized_value(XmlVersion::Implicit1_0, decoder) else { return false; }; @@ -751,14 +749,22 @@ fn esp_system_fields(xml: &str) -> Option { if is_direct_system_path(&path) { match name.as_slice() { b"EventID" => { - let value = - reader.read_text(QName(b"EventID")).ok()?.decode().ok()?.into_owned(); + let value = reader + .read_text(QName(b"EventID")) + .ok()? + .decode() + .ok()? + .into_owned(); fields.event_id.get_or_insert(value); continue; } b"Channel" => { - let value = - reader.read_text(QName(b"Channel")).ok()?.decode().ok()?.into_owned(); + let value = reader + .read_text(QName(b"Channel")) + .ok()? + .decode() + .ok()? + .into_owned(); fields.channel.get_or_insert(value); continue; } diff --git a/src-tauri/src/jamf/connect.rs b/src-tauri/src/jamf/connect.rs index eb30e8cdb..cd8b2ae1e 100644 --- a/src-tauri/src/jamf/connect.rs +++ b/src-tauri/src/jamf/connect.rs @@ -31,9 +31,7 @@ fn user_regex() -> &'static Regex { fn idp_regex() -> &'static Regex { use std::sync::OnceLock; static RE: OnceLock = OnceLock::new(); - RE.get_or_init(|| { - Regex::new(r"provider=(?P\w+)").expect("static regex must compile") - }) + RE.get_or_init(|| Regex::new(r"provider=(?P\w+)").expect("static regex must compile")) } pub fn parse_connect_log_impl(path: &Path) -> Result, AppError> { diff --git a/src-tauri/src/jamf/detect.rs b/src-tauri/src/jamf/detect.rs index 3246b86ad..f996991d5 100644 --- a/src-tauri/src/jamf/detect.rs +++ b/src-tauri/src/jamf/detect.rs @@ -206,7 +206,11 @@ fn read_jss_url() -> Option { return None; } let url = String::from_utf8_lossy(&output.stdout).trim().to_string(); - if url.is_empty() { None } else { Some(url) } + if url.is_empty() { + None + } else { + Some(url) + } } fn read_jamf_connect_version() -> Option { @@ -221,7 +225,11 @@ fn read_jamf_connect_version() -> Option { return None; } let v = String::from_utf8_lossy(&output.stdout).trim().to_string(); - if v.is_empty() { None } else { Some(v) } + if v.is_empty() { + None + } else { + Some(v) + } } fn read_jamf_connect_idp() -> Option { @@ -230,14 +238,11 @@ fn read_jamf_connect_idp() -> Option { continue; } let print_key = format!("Print :{key}"); - let output = match output_with_timeout( - PLISTBUDDY, - &["-c", &print_key, plist], - PLISTBUDDY_TIMEOUT, - ) { - Some(o) => o, - None => continue, - }; + let output = + match output_with_timeout(PLISTBUDDY, &["-c", &print_key, plist], PLISTBUDDY_TIMEOUT) { + Some(o) => o, + None => continue, + }; if !output.status.success() { continue; } @@ -264,6 +269,10 @@ fn build_summary( } else { "" }; - let log = if dirs.jamf_log { "" } else { " · jamf.log missing" }; + let log = if dirs.jamf_log { + "" + } else { + " · jamf.log missing" + }; format!("JAMF binary detected ({v}){connect}{log}") } diff --git a/src-tauri/src/jamf/mod.rs b/src-tauri/src/jamf/mod.rs index b0c5bf102..89b40c7ae 100644 --- a/src-tauri/src/jamf/mod.rs +++ b/src-tauri/src/jamf/mod.rs @@ -1,9 +1,9 @@ +pub mod connect; +pub mod detect; pub mod models; pub mod paths; -pub mod text; -pub mod time; -pub mod detect; pub mod policy_log; -pub mod self_service; -pub mod connect; pub mod profiles; +pub mod self_service; +pub mod text; +pub mod time; diff --git a/src-tauri/src/jamf/models.rs b/src-tauri/src/jamf/models.rs index cd2fa351a..106017621 100644 --- a/src-tauri/src/jamf/models.rs +++ b/src-tauri/src/jamf/models.rs @@ -31,8 +31,8 @@ pub struct JamfDirectoryStatus { pub jamf_log: bool, pub jamf_app_support: bool, pub jamf_receipts: bool, - pub jamf_user_logs: bool, // ~/Library/Logs/JAMF - pub self_service_log: bool, // ~/Library/Logs/JAMF/selfservice.log + pub jamf_user_logs: bool, // ~/Library/Logs/JAMF + pub self_service_log: bool, // ~/Library/Logs/JAMF/selfservice.log pub connect_log: bool, pub connect_user_logs: bool, } diff --git a/src-tauri/src/jamf/paths.rs b/src-tauri/src/jamf/paths.rs index 422bf1cd7..fb93b53f3 100644 --- a/src-tauri/src/jamf/paths.rs +++ b/src-tauri/src/jamf/paths.rs @@ -36,7 +36,10 @@ pub const JAMF_CONNECT_IDP_SOURCES: &[(&str, &str)] = &[ ), ("/Library/Preferences/com.jamf.connect.plist", "Provider"), // Legacy: the key this module originally queried. - ("/Library/Preferences/com.jamf.connect.plist", "OIDCProvider"), + ( + "/Library/Preferences/com.jamf.connect.plist", + "OIDCProvider", + ), ]; /// Returns the installed JAMF Connect app bundle, preferring the modern name. diff --git a/src-tauri/src/jamf/policy_log.rs b/src-tauri/src/jamf/policy_log.rs index 61825d3a5..8d91d2949 100644 --- a/src-tauri/src/jamf/policy_log.rs +++ b/src-tauri/src/jamf/policy_log.rs @@ -234,7 +234,11 @@ fn classify( return ( JamfPolicyTrigger::Other("install".to_string()), None, - Some(pkg.trim_end_matches("...").trim_end_matches('.').to_string()), + Some( + pkg.trim_end_matches("...") + .trim_end_matches('.') + .to_string(), + ), JamfPolicyResult::InProgress, ); } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 052a702ed..111c2a638 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -16,9 +16,9 @@ pub mod intune; #[cfg(debug_assertions)] mod ipc_bridge; #[cfg(feature = "macos-diag")] -pub mod macos_diag; -#[cfg(feature = "macos-diag")] pub mod jamf; +#[cfg(feature = "macos-diag")] +pub mod macos_diag; mod menu; pub use cmtraceopen_parser::models; pub mod parser; @@ -353,6 +353,18 @@ pub fn run() { event_log::commands::evtx_enumerate_channels, #[cfg(feature = "event-log")] event_log::commands::evtx_query_channels, + #[cfg(feature = "event-log")] + event_log::commands::evtx_export_records, + #[cfg(feature = "event-log")] + event_log::commands::evtx_load_event_maps, + #[cfg(feature = "event-log")] + event_log::commands::evtx_loaded_map_count, + #[cfg(feature = "event-log")] + event_log::commands::evtx_load_provider_databases, + #[cfg(feature = "event-log")] + event_log::commands::evtx_provider_databases, + #[cfg(feature = "event-log")] + event_log::commands::evtx_build_unified_timeline, #[cfg(target_os = "windows")] commands::graph_api::graph_authenticate, #[cfg(target_os = "windows")] diff --git a/src-tauri/src/state/app_state.rs b/src-tauri/src/state/app_state.rs index a87cac1a8..50d3d1265 100644 --- a/src-tauri/src/state/app_state.rs +++ b/src-tauri/src/state/app_state.rs @@ -1,11 +1,15 @@ use std::collections::HashMap; use std::path::PathBuf; -#[cfg(feature = "esp-diagnostics")] +#[cfg(any(feature = "esp-diagnostics", feature = "event-log"))] use std::sync::Arc; use std::sync::Mutex; +#[cfg(feature = "event-log")] +use std::sync::RwLock; #[cfg(feature = "esp-diagnostics")] use crate::esp::session::{EspSessionError, EspSessionManager}; +#[cfg(feature = "event-log")] +use crate::event_log::provider_db::ProviderStore; use crate::parser::ResolvedParser; #[cfg(feature = "sccm-diagnostics")] use crate::sccm::collector::SccmAdvancedCapabilityStore; @@ -46,6 +50,18 @@ pub struct AppState { /// its worker and AppHandle-backed event sink cannot outlive the runtime. #[cfg(feature = "esp-diagnostics")] esp_session_manager: Mutex>>, + /// Event maps loaded from disk, applied while rendering event rows. + /// + /// Behind an `Arc>` rather than a `Mutex` on the state itself so a command can take + /// a cheap handle and carry it into `spawn_blocking`. Parsing a hundred thousand records is + /// exactly the blocking work that must not run while the application state lock is held. + #[cfg(feature = "event-log")] + pub event_maps: Arc>, + /// Provider metadata databases, read to render an event's own description. + /// + /// Held the same way and for the same reason as [`event_maps`](Self::event_maps). + #[cfg(feature = "event-log")] + pub provider_store: Arc>, } impl AppState { @@ -69,6 +85,10 @@ impl AppState { sccm_advanced_capabilities: Mutex::new(SccmAdvancedCapabilityStore::default()), #[cfg(feature = "esp-diagnostics")] esp_session_manager: Mutex::new(None), + #[cfg(feature = "event-log")] + event_maps: Arc::new(RwLock::new(cmtraceopen_parser::eventmap::MapRegistry::new())), + #[cfg(feature = "event-log")] + provider_store: Arc::new(RwLock::new(ProviderStore::default())), } } diff --git a/src-tauri/src/watcher/tail.rs b/src-tauri/src/watcher/tail.rs index bbbff2530..120d0daa0 100644 --- a/src-tauri/src/watcher/tail.rs +++ b/src-tauri/src/watcher/tail.rs @@ -438,20 +438,19 @@ impl TailReader { } if !lines.is_empty() { - let remaining_start = if let Some(initial) = - self.consume_initial_company_portal_lines(&lines, now) - { - let remaining_start = initial.remaining_start; - batch.append(initial.batch); - if remaining_start.is_none() { - batch.append(self.enforce_initial_fragment_bound(now)); - self.byte_offset = file_size; - return Ok(batch); - } - remaining_start.unwrap_or_default() - } else { - 0 - }; + let remaining_start = + if let Some(initial) = self.consume_initial_company_portal_lines(&lines, now) { + let remaining_start = initial.remaining_start; + batch.append(initial.batch); + if remaining_start.is_none() { + batch.append(self.enforce_initial_fragment_bound(now)); + self.byte_offset = file_size; + return Ok(batch); + } + remaining_start.unwrap_or_default() + } else { + 0 + }; let prior = self .pending_logical_record @@ -1100,7 +1099,11 @@ impl TailReader { self.pending_fragment_selection = Some((selection.clone(), now)); } - self.parse_company_portal_records(framed.completed_records, selection, framed.overflow_count) + self.parse_company_portal_records( + framed.completed_records, + selection, + framed.overflow_count, + ) } fn parse_company_portal_records( diff --git a/src-tauri/tests/event_log_real_evtx.rs b/src-tauri/tests/event_log_real_evtx.rs new file mode 100644 index 000000000..0033c7034 --- /dev/null +++ b/src-tauri/tests/event_log_real_evtx.rs @@ -0,0 +1,170 @@ +//! Runs the .evtx file path against a real Windows event log. +//! +//! The unit tests in `event_log::parser` cover field extraction against hand-written XML. They +//! cannot cover the part that actually broke: what the `evtx` crate hands back per record, and +//! whether that survives the whole pipeline. The file path once fed a JSON projection to an XML +//! parser, which failed silently for every record and left the System block, every map, and the +//! XML export empty on an opened file. Nothing in the unit tests could see that. +//! +//! Captured logs carry real hostnames, account names and query traffic, so they live outside the +//! repo. Point `CMTRACE_EVTX_FIXTURE` at an .evtx file to run these; without it every test here +//! passes vacuously, so CI is unaffected. +//! +//! CMTRACE_EVTX_FIXTURE=~/logs/dns-audit.evtx cargo test --features event-log --test event_log_real_evtx +//! +//! Assertions are floors and invariants rather than exact counts, so a different capture does not +//! break them. + +#![cfg(feature = "event-log")] + +use std::path::PathBuf; + +use app_lib::event_log::export::{export_records, ExportFormat}; +use app_lib::event_log::models::EvtxParseResult; +use app_lib::event_log::parser::parse_evtx_files; +use app_lib::event_log::provider_db::ProviderStore; +use cmtraceopen_parser::eventmap::MapRegistry; +use std::sync::RwLock; + +fn fixture() -> Option { + let Some(raw) = std::env::var_os("CMTRACE_EVTX_FIXTURE") else { + // Said out loud rather than passing silently. Seven tests reporting ok with nothing run is + // the same failure this suite exists to catch: an empty result that reads as a verified + // one. Visible with `cargo test -- --nocapture`, and in CI logs. + eprintln!( + "SKIP: CMTRACE_EVTX_FIXTURE is not set, so nothing in this file actually ran. \ + Point it at an .evtx file to exercise the real parse path." + ); + return None; + }; + let path = PathBuf::from(raw); + assert!( + path.is_file(), + "CMTRACE_EVTX_FIXTURE is set but {} is not a file", + path.display() + ); + Some(path) +} + +fn parsed() -> Option { + let path = fixture()?; + // Registries local to this call. The parse path takes them explicitly, so nothing here can be + // perturbed by another test loading a different set on a parallel thread. + let maps = RwLock::new(MapRegistry::new()); + let providers = RwLock::new(ProviderStore::default()); + let result = parse_evtx_files(&[path.to_string_lossy().into_owned()], &maps, &providers) + .expect("the file parses"); + assert!( + !result.records.is_empty(), + "fixture produced no records at all" + ); + Some(result) +} + +#[test] +fn every_record_in_the_file_parses() { + let Some(result) = parsed() else { return }; + // A record that cannot be read is counted, not dropped in silence. A real log from a healthy + // machine should have none, so any error here means the reader is wrong rather than the file. + assert_eq!( + result.parse_errors, + 0, + "{} of {} records failed to parse", + result.parse_errors, + result.records.len() + ); +} + +#[test] +fn identity_is_populated_rather_than_defaulted() { + let Some(result) = parsed() else { return }; + for record in &result.records { + assert_ne!( + record.provider, "Unknown", + "record {} has no provider", + record.event_record_id + ); + assert_ne!( + record.channel, "Unknown", + "record {} has no channel", + record.event_record_id + ); + // Event ID 0 is legal and several in-box providers emit it, so it is not an invariant. + assert_ne!( + record.timestamp_epoch, 0, + "record {} sorted to the epoch, so the timeline order is wrong", + record.event_record_id + ); + } +} + +#[test] +fn the_system_block_survives_the_file_path() { + let Some(result) = parsed() else { return }; + // These were empty on every opened file while the path re-parsed JSON as XML. Asserting that + // some record carries each one, rather than all of them, because providers legitimately omit + // any individual field. + let has = |f: fn(&app_lib::event_log::models::EvtxRecord) -> bool| result.records.iter().any(f); + assert!(has(|r| r.process_id.is_some()), "no record carries a PID"); + assert!(has(|r| r.thread_id.is_some()), "no record carries a TID"); + assert!(has(|r| r.keywords.is_some()), "no record carries keywords"); +} + +#[test] +fn records_carry_event_fields() { + let Some(result) = parsed() else { return }; + assert!( + result.records.iter().any(|r| !r.event_data.is_empty()), + "no record has any fields, so EventData and UserData were both missed" + ); + for record in &result.records { + for field in &record.event_data { + assert!(!field.name.is_empty(), "a field was extracted with no name"); + } + } +} + +#[test] +fn the_xml_export_emits_the_provider_representation() { + let Some(result) = parsed() else { return }; + // The export used to contain pretty-printed JSON under an root, which no XML consumer + // could read. + let exported = export_records(&result.records[..1], ExportFormat::Xml).expect("exports"); + assert!( + exported.contains(" = result + .error_messages + .iter() + .filter(|message| !message.contains("stopped at")) + .collect(); + assert!( + unexpected.is_empty(), + "clean capture reported gaps: {unexpected:?}" + ); + assert_eq!(result.total_records, result.records.len() as u64); +} + +#[test] +fn records_are_ordered_by_time() { + let Some(result) = parsed() else { return }; + let ordered = result + .records + .windows(2) + .all(|pair| pair[0].timestamp_epoch <= pair[1].timestamp_epoch); + assert!(ordered, "records are not sorted by timestamp"); +} diff --git a/src-tauri/tests/graph_esp_diagnostics.rs b/src-tauri/tests/graph_esp_diagnostics.rs index afd08dfd2..541e60deb 100644 --- a/src-tauri/tests/graph_esp_diagnostics.rs +++ b/src-tauri/tests/graph_esp_diagnostics.rs @@ -228,7 +228,12 @@ fn graph_permission_upgrade_command_runs_owned_wam_on_a_blocking_worker() { "pub(crate) fn authenticate(", "pub(crate) fn request_missing_permissions(", ); - assert_eq!(authenticate.matches("wam::authentication_deadline()").count(), 1); + assert_eq!( + authenticate + .matches("wam::authentication_deadline()") + .count(), + 1 + ); assert!(authenticate.contains("match wam::acquire_token(hwnd_raw, deadline, lease)")); assert!(authenticate.contains("probe_host_capability_for_authentication(deadline, lease)")); assert!(graph_source.contains("std::time::Duration::from_secs(120)")); diff --git a/src-tauri/tests/jamf_environment.rs b/src-tauri/tests/jamf_environment.rs index e66b50f8d..6fb11c576 100644 --- a/src-tauri/tests/jamf_environment.rs +++ b/src-tauri/tests/jamf_environment.rs @@ -9,7 +9,10 @@ fn collect_environment_does_not_panic() { // An installed binary does not guarantee `jamf version` succeeds (it can // fail or time out), so only the inverse is a real invariant. if !env.jamf_installed { - assert!(env.jamf_version.is_none(), "version reported without a binary"); + assert!( + env.jamf_version.is_none(), + "version reported without a binary" + ); } } diff --git a/src-tauri/tests/jamf_ipc_contract.rs b/src-tauri/tests/jamf_ipc_contract.rs index 53d41267c..4a356e1f6 100644 --- a/src-tauri/tests/jamf_ipc_contract.rs +++ b/src-tauri/tests/jamf_ipc_contract.rs @@ -45,7 +45,10 @@ fn policy_trigger_data_variants_carry_a_value_field() { #[test] fn policy_result_matches_the_typescript_union() { assert_eq!(json(&JamfPolicyResult::Success), r#"{"type":"success"}"#); - assert_eq!(json(&JamfPolicyResult::InProgress), r#"{"type":"inProgress"}"#); + assert_eq!( + json(&JamfPolicyResult::InProgress), + r#"{"type":"inProgress"}"# + ); assert_eq!(json(&JamfPolicyResult::Unknown), r#"{"type":"unknown"}"#); assert_eq!( json(&JamfPolicyResult::Failure("Error running recon".into())), diff --git a/src-tauri/tests/jamf_known_sources.rs b/src-tauri/tests/jamf_known_sources.rs index e770c9979..4c43778d0 100644 --- a/src-tauri/tests/jamf_known_sources.rs +++ b/src-tauri/tests/jamf_known_sources.rs @@ -22,7 +22,10 @@ fn jamf_known_sources_present() { let all = build_known_log_sources(); let jamf = jamf_sources(&all); let ids: Vec<&str> = jamf.iter().map(|s| s.id.as_str()).collect(); - assert!(ids.contains(&"macos-jamf-log"), "missing macos-jamf-log: {ids:?}"); + assert!( + ids.contains(&"macos-jamf-log"), + "missing macos-jamf-log: {ids:?}" + ); assert!(ids.contains(&"macos-jamf-app-support-logs")); assert!(ids.contains(&"macos-jamf-receipts")); assert!(ids.contains(&"macos-jamf-self-service-log")); @@ -40,11 +43,11 @@ fn jamf_log_default_file_is_jamf_log() { .iter() .find(|s| s.id == "macos-jamf-log") .expect("macos-jamf-log should exist"); - let intent = entry.default_file_intent.as_ref().expect("should have default file intent"); - assert!(intent - .preferred_file_names - .iter() - .any(|n| n == "jamf.log")); + let intent = entry + .default_file_intent + .as_ref() + .expect("should have default file intent"); + assert!(intent.preferred_file_names.iter().any(|n| n == "jamf.log")); } #[cfg(not(target_os = "macos"))] diff --git a/src-tauri/tests/jamf_parser_robustness.rs b/src-tauri/tests/jamf_parser_robustness.rs index 7bed41c27..1bba452c9 100644 --- a/src-tauri/tests/jamf_parser_robustness.rs +++ b/src-tauri/tests/jamf_parser_robustness.rs @@ -32,7 +32,11 @@ fn policy_log_survives_invalid_utf8_and_keeps_all_lines() { let result = parse_policy_log_impl(&path).expect("must not fail on invalid UTF-8"); assert_eq!(result.total_lines, 3); - assert_eq!(result.events.len(), 3, "every line must still yield an event"); + assert_eq!( + result.events.len(), + 3, + "every line must still yield an event" + ); assert_eq!(result.unparsed_lines, 0); let names: Vec<&str> = result @@ -42,7 +46,11 @@ fn policy_log_survives_invalid_utf8_and_keeps_all_lines() { .collect(); assert_eq!(names[0], "First"); assert_eq!(names[2], "Third"); - assert!(names[1].starts_with("Ba"), "decoded lossily: {:?}", names[1]); + assert!( + names[1].starts_with("Ba"), + "decoded lossily: {:?}", + names[1] + ); let _ = std::fs::remove_file(&path); } @@ -77,7 +85,10 @@ fn policy_log_tolerates_a_truncated_final_line() { let result = parse_policy_log_impl(&path).expect("parse"); assert_eq!(result.total_lines, 2); assert_eq!(result.events.len(), 1); - assert_eq!(result.unparsed_lines, 1, "the partial line is counted, not fatal"); + assert_eq!( + result.unparsed_lines, 1, + "the partial line is counted, not fatal" + ); let _ = std::fs::remove_file(&path); } @@ -190,9 +201,11 @@ fn an_over_long_line_does_not_disturb_later_offsets() { fn missing_files_are_not_errors_except_for_the_policy_log() { // Self Service / Connect may legitimately be absent; jamf.log going missing // is a condition worth surfacing. - assert!(parse_self_service_log_impl(Path::new("/nonexistent/ss.log")) - .expect("absent Self Service log is empty, not an error") - .is_empty()); + assert!( + parse_self_service_log_impl(Path::new("/nonexistent/ss.log")) + .expect("absent Self Service log is empty, not an error") + .is_empty() + ); assert!(parse_connect_log_impl(Path::new("/nonexistent/jc.log")) .expect("absent Connect log is empty, not an error") .is_empty()); diff --git a/src-tauri/tests/jamf_policy_log_parsing.rs b/src-tauri/tests/jamf_policy_log_parsing.rs index fb1c33d53..aac7fb35d 100644 --- a/src-tauri/tests/jamf_policy_log_parsing.rs +++ b/src-tauri/tests/jamf_policy_log_parsing.rs @@ -77,7 +77,10 @@ fn classifies_jss_connectivity_failure() { #[test] fn unparsed_lines_counted() { let result = parse_policy_log_impl(Path::new(BASIC)).expect("parse"); - assert_eq!(result.total_lines, result.events.len() + result.unparsed_lines); + assert_eq!( + result.total_lines, + result.events.len() + result.unparsed_lines + ); } #[test] @@ -129,8 +132,10 @@ Wed Jul 22 20:10:32 host jamf[6003]: Successfully installed Zscaler-osx-4.5.2.31 let installing = result .events .iter() - .find(|e| matches!(&e.result, JamfPolicyResult::InProgress) - && e.policy_name.as_deref() == Some("Zscaler-osx-4.5.2.312-installer.pkg")) + .find(|e| { + matches!(&e.result, JamfPolicyResult::InProgress) + && e.policy_name.as_deref() == Some("Zscaler-osx-4.5.2.312-installer.pkg") + }) .expect("the Installing line should name the package"); assert!(matches!(&installing.trigger, JamfPolicyTrigger::Other(k) if k == "install")); diff --git a/src-tauri/tests/jamf_real_fixtures.rs b/src-tauri/tests/jamf_real_fixtures.rs index 575b90168..efe9bd60d 100644 --- a/src-tauri/tests/jamf_real_fixtures.rs +++ b/src-tauri/tests/jamf_real_fixtures.rs @@ -117,7 +117,10 @@ fn self_service_log_yields_user_actions() { let Some(dir) = fixture_dir() else { return }; let events = parse_self_service_log_impl(&dir.join("logs/selfservice.log")).expect("parse"); - assert!(!events.is_empty(), "real selfservice.log produced no events"); + assert!( + !events.is_empty(), + "real selfservice.log produced no events" + ); assert!( events.iter().any(|e| e.action == "triggerPolicy"), "capture is known to contain Self Service-initiated installs" @@ -170,8 +173,8 @@ fn captured_profiles_xml_parses_and_filters_to_jamf() { profiles.len() ); - let filtered = app_lib::jamf::profiles::filter_jamf_profiles_impl(profiles, None) - .expect("filter"); + let filtered = + app_lib::jamf::profiles::filter_jamf_profiles_impl(profiles, None).expect("filter"); assert!( !filtered.profiles.is_empty(), "payload-prefix matching found no JAMF profiles in a JAMF capture" diff --git a/src-tauri/tests/jamf_self_service_log_parsing.rs b/src-tauri/tests/jamf_self_service_log_parsing.rs index ada433488..b33283c99 100644 --- a/src-tauri/tests/jamf_self_service_log_parsing.rs +++ b/src-tauri/tests/jamf_self_service_log_parsing.rs @@ -28,7 +28,10 @@ fn parses_real_selfservice_log_shapes() { // API chatter is one action with the endpoint as the item. assert_eq!(events[4].action, "request"); - assert_eq!(events[4].item_name.as_deref(), Some("updateDevicePushToken")); + assert_eq!( + events[4].item_name.as_deref(), + Some("updateDevicePushToken") + ); assert_eq!(events[7].action, "warning"); assert!(events[7] @@ -37,7 +40,10 @@ fn parses_real_selfservice_log_shapes() { .is_some_and(|m| m.starts_with("A customized icon"))); // Binary requests are the user-side operations. - let triggers: Vec<&_> = events.iter().filter(|e| e.action == "triggerPolicy").collect(); + let triggers: Vec<&_> = events + .iter() + .filter(|e| e.action == "triggerPolicy") + .collect(); assert_eq!(triggers.len(), 2); assert_eq!(events.iter().filter(|e| e.action == "doRecon").count(), 1); } diff --git a/src/workspaces/event-log/EventLogWorkspace.tsx b/src/workspaces/event-log/EventLogWorkspace.tsx index 285c6890a..bfd47cb12 100644 --- a/src/workspaces/event-log/EventLogWorkspace.tsx +++ b/src/workspaces/event-log/EventLogWorkspace.tsx @@ -4,6 +4,7 @@ import { useEvtxStore } from "./evtx-store"; import { SourcePicker } from "./SourcePicker"; import { ChannelPicker } from "./ChannelPicker"; import { EvtxFilterBar } from "./EvtxFilterBar"; +import { EvtxCoverageBanner } from "./EvtxCoverageBanner"; import { EvtxTimeline } from "./EvtxTimeline"; import { EvtxDetailPane } from "./EvtxDetailPane"; @@ -85,6 +86,7 @@ export function EventLogWorkspace() { )} +
s.coverageGaps); + const logListFontSize = useUiStore((s) => s.logListFontSize); + const [collapsed, setCollapsed] = useState(false); + + const { fontSize, rowLineHeight } = getLogListMetrics(logListFontSize); + const summary = summarizeCoverageGaps(gaps); + + // The live region is always rendered, and the banner content appears inside it. A screen reader + // announces changes within a region it was already tracking, so a region that arrives already + // populated is read as ordinary page content and the first gaps go unannounced. It must also + // stay in the accessibility tree while empty, which display:none would prevent, so an empty + // region is simply an unstyled element with no children. + const empty = gaps.length === 0; + + return ( +
+ {empty ? null : ( + <> +
+ {summary} + +
+ {!collapsed && ( +
    + {gaps.map((gap) => ( +
  • + {gap} +
    • + ))} +
+ )} + + )} +
+ ); +} diff --git a/src/workspaces/event-log/EvtxDetailPane.tsx b/src/workspaces/event-log/EvtxDetailPane.tsx index f12a5fe3b..e6dd33121 100644 --- a/src/workspaces/event-log/EvtxDetailPane.tsx +++ b/src/workspaces/event-log/EvtxDetailPane.tsx @@ -198,8 +198,91 @@ export function EvtxDetailPane() { Source: {record.sourceLabel} + {/* System-block fields. Rendered only when the provider actually wrote them, so an absent + value reads as absent rather than as a zero the provider never claimed. */} + {record.task != null && ( + + Task: {record.task} + + )} + {record.opcode != null && ( + + Opcode: {record.opcode} + + )} + {record.processId != null && ( + + PID: {record.processId} + + )} + {record.threadId != null && ( + + TID: {record.threadId} + + )} + {record.keywords && ( + + Keywords: {record.keywords} + + )} + {record.userSid && ( + + User SID: {record.userSid} + + )}
+ {/* Map-derived columns. Only present where a map covers this event type, so the section is + hidden entirely rather than showing an empty heading. */} + {record.mapped && record.mapped.length > 0 && ( +
+
+ Mapped fields +
+ {record.mapped.map((column) => ( +
+ + {column.property} + + + {column.text} + +
+ ))} +
+ )} + {/* Raw XML */}
+ +
+ {LEVELS.map((level) => { const active = filterLevels.has(level); return ( @@ -72,7 +275,7 @@ export function EvtxFilterBar() { style={{ minWidth: "auto", padding: "2px 8px", - fontSize: "11px", + fontSize: controlFontSize, borderColor: active ? undefined : LEVEL_COLORS[level], color: active ? undefined : LEVEL_COLORS[level], }} @@ -99,6 +302,178 @@ export function EvtxFilterBar() { style={{ width: "160px" }} /> + { + const id = data.optionValue as EvtxColumnId | undefined; + if (id) toggleColumnVisible(id); + }} + > + {choosableColumns.map((column) => ( + + ))} + + + {/* + A button, not an option. In a multiselect listbox every option carries a selection + indicator, so "Reset to defaults" rendered as though it were a column that could be + checked, and it is an action rather than a member of the set. + */} + + + {/* + Reordering lives outside the listbox. Buttons nested in a Fluent Option are invalid ARIA + and never receive focus, because a listbox moves focus between options rather than into + them, so a keyboard-only operator could show and hide columns but never order them. + */} + { + if (data.optionValue) setReorderTarget(data.optionValue as EvtxColumnId); + }} + > + {columnConfig.order.map((id) => ( + + ))} + + + + + + { + if (data.optionValue === "__save__") setPendingName(""); + else if (data.optionValue) applySavedFilter(data.optionValue); + }} + > + + {orderedFilters.map((filter) => ( + + ))} + + + EVTX_GROUP_LABELS[field]).join(" > ")} + selectedOptions={groupBy} + style={{ minWidth: "128px" }} + title="Group the list. Selecting several nests them in the order chosen." + onOptionSelect={(_, data) => { + const field = data.optionValue as EvtxGroupField; + if (!field) return; + // Appending rather than sorting keeps the nesting order under the operator's control. + setGroupBy( + groupBy.includes(field) + ? groupBy.filter((existing) => existing !== field) + : [...groupBy, field] + ); + }} + > + {GROUP_FIELDS.map((field) => ( + + ))} + + + { + const format = EXPORT_FORMATS.find((f) => f.value === data.optionValue); + if (format) void exportVisible(format); + }} + > + {EXPORT_FORMATS.map((format) => ( + + ))} + + + {pendingName !== null && ( + setPendingName(data.value)} + onKeyDown={(event) => { + if (event.key === "Enter") commitFilterName(pendingName); + if (event.key === "Escape") setPendingName(null); + }} + onBlur={() => setPendingName(null)} + /> + )} + + {exportState && ( + + {exportState} + + )} + setFilterSearch(data.value)} @@ -112,7 +487,7 @@ export function EvtxFilterBar() {
diff --git a/src/workspaces/event-log/EvtxTimeline.tsx b/src/workspaces/event-log/EvtxTimeline.tsx index 79fd8f666..bf9820755 100644 --- a/src/workspaces/event-log/EvtxTimeline.tsx +++ b/src/workspaces/event-log/EvtxTimeline.tsx @@ -7,7 +7,13 @@ import { } from "../../lib/log-accessibility"; import { useUiStore } from "../../stores/ui-store"; import { useEvtxStore, type EvtxSortField } from "./evtx-store"; +import { + parseEventIdFilter, + buildGroupedRows, + type EvtxRow, +} from "./evtx-filter"; import type { EvtxRecord, EvtxLevel } from "./types"; +import { visibleColumns } from "./evtx-columns"; import { EvtxTimelineRow } from "./EvtxTimelineRow"; const LEVEL_ORDER: Record = { @@ -45,17 +51,6 @@ function compareRecords( return direction === "asc" ? cmp : -cmp; } -function parseEventIdFilter(raw: string): Set | null { - const trimmed = raw.trim(); - if (!trimmed) return null; - const ids = new Set(); - for (const part of trimmed.split(",")) { - const n = parseInt(part.trim(), 10); - if (!isNaN(n)) ids.add(n); - } - return ids.size > 0 ? ids : null; -} - export function EvtxTimeline() { const records = useEvtxStore((s) => s.records); const selectedChannels = useEvtxStore((s) => s.selectedChannels); @@ -64,6 +59,11 @@ export function EvtxTimeline() { const filterSearch = useEvtxStore((s) => s.filterSearch); const sortField = useEvtxStore((s) => s.sortField); const sortDirection = useEvtxStore((s) => s.sortDirection); + const groupBy = useEvtxStore((s) => s.groupBy); + const collapsedGroups = useEvtxStore((s) => s.collapsedGroups); + const timeZoneMode = useEvtxStore((s) => s.timeZoneMode); + const toggleGroup = useEvtxStore((s) => s.toggleGroup); + const columnConfig = useEvtxStore((s) => s.columnConfig); const selectedRecordId = useEvtxStore((s) => s.selectedRecordId); const setSelectedRecordId = useEvtxStore((s) => s.setSelectedRecordId); @@ -103,21 +103,51 @@ export function EvtxTimeline() { ); }, [filteredRecords, sortField, sortDirection]); - useEffect(() => { - if (selectedRecordId == null) return; - const stillVisible = sortedRecords.some((r) => r.id === selectedRecordId); - if (!stillVisible) { - setSelectedRecordId(null); - } - }, [sortedRecords, setSelectedRecordId, selectedRecordId]); - const parentRef = useRef(null); + // Grouping produces header rows interleaved with records, so the virtualizer indexes rows rather + // than records. With no grouping the row list is the record list and nothing changes. + // Computed once rather than per row: columnConfig is stable between renders, and the row + // renderer was rebuilding the spec array and re-synthesizing every map column spec for each of + // potentially a hundred thousand rows. + const columns = useMemo(() => visibleColumns(columnConfig), [columnConfig]); + + const rows: EvtxRow[] = useMemo( + () => buildGroupedRows(sortedRecords, groupBy, collapsedGroups, timeZoneMode), + [sortedRecords, groupBy, collapsedGroups, timeZoneMode] + ); + + // Keyboard navigation moves between records, skipping headers, because a header is not a + // selectable event. + const recordRowIndexes = useMemo( + () => + rows.reduce((indexes, row, index) => { + if (row.kind === "record") indexes.push(index); + return indexes; + }, []), + [rows] + ); + + // Checked against the rendered rows, not the filtered records. Filtering already dropped a + // hidden selection, but collapsing a group leaves the record in sortedRecords while taking its + // row out of the list, so keyboard navigation could not find the current position and the + // detail pane showed a record that was not on screen. Rows covers both. + useEffect(() => { + if (selectedRecordId === null) return; + const stillVisible = rows.some( + (row) => row.kind === "record" && row.record.id === selectedRecordId + ); + if (!stillVisible) setSelectedRecordId(null); + }, [rows, selectedRecordId, setSelectedRecordId]); + const virtualizer = useVirtualizer({ - count: sortedRecords.length, + count: rows.length, getScrollElement: () => parentRef.current, estimateSize: () => rowEstimate, - getItemKey: (index) => sortedRecords[index]?.id ?? index, + getItemKey: (index) => { + const row = rows[index]; + return row?.kind === "group" ? `group:${row.key}` : row?.record.id ?? index; + }, overscan: 10, }); @@ -134,29 +164,37 @@ export function EvtxTimeline() { e.preventDefault(); e.stopPropagation(); - const currentIndex = selectedRecordId != null - ? sortedRecords.findIndex((r) => r.id === selectedRecordId) + if (recordRowIndexes.length === 0) return; + + const currentPosition = selectedRecordId != null + ? recordRowIndexes.findIndex((rowIndex) => { + const row = rows[rowIndex]; + return row.kind === "record" && row.record.id === selectedRecordId; + }) : -1; - let nextIndex: number; + let nextPosition: number; if (e.key === "ArrowDown") { - nextIndex = currentIndex < sortedRecords.length - 1 ? currentIndex + 1 : currentIndex; + nextPosition = currentPosition < recordRowIndexes.length - 1 ? currentPosition + 1 : currentPosition; } else if (e.key === "ArrowUp") { - nextIndex = currentIndex > 0 ? currentIndex - 1 : 0; + nextPosition = currentPosition > 0 ? currentPosition - 1 : 0; } else if (e.key === "Home") { - nextIndex = 0; + nextPosition = 0; } else { - nextIndex = sortedRecords.length - 1; + nextPosition = recordRowIndexes.length - 1; } - if (nextIndex >= 0 && nextIndex < sortedRecords.length) { - setSelectedRecordId(sortedRecords[nextIndex].id); - virtualizer.scrollToIndex(nextIndex, { align: "auto" }); + if (nextPosition < 0) nextPosition = 0; + const rowIndex = recordRowIndexes[nextPosition]; + const row = rows[rowIndex]; + if (row?.kind === "record") { + setSelectedRecordId(row.record.id); + virtualizer.scrollToIndex(rowIndex, { align: "auto" }); // Keep focus on the container so subsequent arrow keys work parentRef.current?.focus(); } }, - [selectedRecordId, sortedRecords, setSelectedRecordId, virtualizer] + [selectedRecordId, rows, recordRowIndexes, setSelectedRecordId, virtualizer] ); if (records.length === 0) { @@ -194,7 +232,10 @@ export function EvtxTimeline() { return (
0 ? "tree" : "listbox"} tabIndex={0} onKeyDown={handleKeyDown} aria-label={`Event log timeline - ${sortedRecords.length} records`} @@ -224,7 +265,56 @@ export function EvtxTimeline() { }} > {virtualRows.map((virtualRow) => { - const record = sortedRecords[virtualRow.index]; + const row = rows[virtualRow.index]; + if (!row) return null; + + if (row.kind === "group") { + return ( +
toggleGroup(row.key)} + onKeyDown={(event) => { + if (event.key === "Enter" || event.key === " ") { + event.preventDefault(); + toggleGroup(row.key); + } + }} + style={{ + // Normal flow, matching the record rows. The wrapper above is already + // translated to the first visible row's offset and its children stack inside + // it, so positioning a header absolutely applied that offset a second time and + // took it out of flow, letting the rows beneath slide up into its place. + width: "100%", + display: "flex", + alignItems: "center", + gap: "6px", + paddingLeft: `${8 + row.depth * 16}px`, + height: `${metrics.rowHeight}px`, + fontSize: `${smallFontSize}px`, + fontWeight: 600, + cursor: "pointer", + backgroundColor: tokens.colorNeutralBackground3, + borderBottom: `1px solid ${tokens.colorNeutralStroke2}`, + color: tokens.colorNeutralForeground2, + }} + title={`${row.count} events`} + > + {row.collapsed ? "\u25B8" : "\u25BE"} + {row.label} + ({row.count}) +
+ ); + } + + const record = row.record; return ( ); diff --git a/src/workspaces/event-log/EvtxTimelineRow.tsx b/src/workspaces/event-log/EvtxTimelineRow.tsx index 27f6baa55..7a10af21b 100644 --- a/src/workspaces/event-log/EvtxTimelineRow.tsx +++ b/src/workspaces/event-log/EvtxTimelineRow.tsx @@ -4,6 +4,13 @@ import { LOG_MONOSPACE_FONT_FAMILY, } from "../../lib/log-accessibility"; import type { EvtxRecord, EvtxLevel } from "./types"; +import { + columnValue, + columnWidth, + type EvtxColumnConfig, + type EvtxColumnSpec, +} from "./evtx-columns"; +import type { EvtxTimeZoneMode } from "./evtx-time"; const LEVEL_COLORS: Record = { Critical: tokens.colorPaletteRedForeground1, @@ -29,6 +36,11 @@ export interface EvtxTimelineRowProps { smallFontSize: number; monoFontSize: number; lineHeight: string; + columnConfig: EvtxColumnConfig; + /** Precomputed by the list, since columnConfig is stable and this renders once per row. */ + columns: EvtxColumnSpec[]; + /** Passed rather than read from the store, so a memoized row re-renders when the clock changes. */ + timeZoneMode: EvtxTimeZoneMode; onSelect: (id: number | null) => void; } @@ -42,6 +54,9 @@ export const EvtxTimelineRow = memo( smallFontSize, monoFontSize, lineHeight, + columnConfig, + columns, + timeZoneMode, onSelect, }, ref @@ -82,105 +97,75 @@ export const EvtxTimelineRow = memo( minWidth: 0, }} > - {/* Level badge */} -
- {LEVEL_SHORT[record.level]} -
- - {/* Timestamp */} -
- {record.timestamp} -
+ {columns.map((column) => { + const width = columnWidth(columnConfig, column); + const value = columnValue(record, column.id, timeZoneMode); - {/* Event ID */} -
- {record.eventId} -
- - {/* Channel badge */} -
- {record.channel} -
+ if (column.id === "level") { + return ( +
+ {LEVEL_SHORT[record.level]} +
+ ); + } - {/* Provider */} -
- {record.provider} -
+ const isDescription = column.id === "message"; + const isMono = column.id === "timestamp" || column.id === "keywords"; - {/* Message preview */} -
- {record.message} -
+ return ( +
+ {value} +
+ ); + })}
); }) diff --git a/src/workspaces/event-log/UnifiedTimelineView.tsx b/src/workspaces/event-log/UnifiedTimelineView.tsx new file mode 100644 index 000000000..e686883cb --- /dev/null +++ b/src/workspaces/event-log/UnifiedTimelineView.tsx @@ -0,0 +1,211 @@ +import { useMemo, useRef } from "react"; +import { tokens } from "@fluentui/react-components"; +import { useVirtualizer } from "@tanstack/react-virtual"; + +import { LOG_MONOSPACE_FONT_FAMILY, LOG_UI_FONT_FAMILY, getLogListMetrics } from "../../lib/log-accessibility"; +import { useUiStore } from "../../stores/ui-store"; +import { formatEventTime } from "./evtx-time"; +import { useEvtxStore } from "./evtx-store"; +import { + isEventOrigin, + originDetail, + originLabel, + timelineCounts, + unplacedSummary, + type TimelineSeverity, + type UnifiedTimeline, +} from "./unified-timeline"; + +const SEVERITY_COLORS: Record = { + critical: tokens.colorPaletteRedForeground1, + error: tokens.colorPaletteRedForeground1, + warning: tokens.colorPaletteMarigoldForeground1, + info: tokens.colorBrandForeground1, + verbose: tokens.colorNeutralForeground4, +}; + +export interface UnifiedTimelineViewProps { + timeline: UnifiedTimeline; +} + +/** + * The merged view of events and text logs. + * + * Rows carry a source badge because the whole value of the view is knowing, at a glance, which + * side of the merge a line came from. Without it the two blur together and the reader loses the + * distinction that makes the correlation meaningful. + */ +export function UnifiedTimelineView({ timeline }: UnifiedTimelineViewProps) { + const timeZoneMode = useEvtxStore((s) => s.timeZoneMode); + const logListFontSize = useUiStore((s) => s.logListFontSize); + const metrics = useMemo(() => getLogListMetrics(logListFontSize), [logListFontSize]); + + const parentRef = useRef(null); + const virtualizer = useVirtualizer({ + count: timeline.items.length, + getScrollElement: () => parentRef.current, + estimateSize: () => metrics.rowHeight + 2, + overscan: 12, + }); + + const counts = useMemo(() => timelineCounts(timeline), [timeline]); + const dropped = useMemo(() => unplacedSummary(timeline), [timeline]); + + const fontSize = metrics.fontSize; + const smallFontSize = Math.max(9, fontSize - 3); + const monoFontSize = Math.max(10, fontSize - 1); + + return ( +
+
+ {counts.logs.toLocaleString()} log lines + {counts.events.toLocaleString()} events + {/* Only shown when something was actually dropped; a "0 unplaced" badge would read as + reassurance and invite no attention. */} + {dropped && ( + + {dropped} + + )} +
+ + {timeline.items.length === 0 ? ( +
+ Nothing to place on the timeline yet. Load a log file and an event source to correlate + them. +
+ ) : ( +
+
+ {virtualizer.getVirtualItems().map((virtualRow) => { + const item = timeline.items[virtualRow.index]; + if (!item) return null; + const color = SEVERITY_COLORS[item.severity]; + const fromEvent = isEventOrigin(item.origin); + + return ( +
+ + {fromEvent ? "EVT" : "LOG"} + + + + {formatEventTime(item.timestampMs, timeZoneMode)} + + + + {originLabel(item.origin)} + + + + {item.message} + +
+ ); + })} +
+
+ )} +
+ ); +} diff --git a/src/workspaces/event-log/evtx-columns.test.ts b/src/workspaces/event-log/evtx-columns.test.ts new file mode 100644 index 000000000..d0696864a --- /dev/null +++ b/src/workspaces/event-log/evtx-columns.test.ts @@ -0,0 +1,253 @@ +import { eventDateKey, formatEventTime } from "./evtx-time"; +import { describe, expect, it } from "vitest"; +import { + availableColumns, + discoverMappedProperties, + mappedColumnId, + columnValue, + columnWidth, + defaultColumnConfig, + EVTX_COLUMNS, + moveColumn, + sanitizeColumnConfig, + type EvtxColumnId, + type EvtxColumnSpec, + toggleColumn, + visibleColumns, + type EvtxColumnConfig, +} from "./evtx-columns"; +import type { EvtxRecord } from "./types"; + +/** + * The spec for `id`, failing with the id when it is absent. + * + * A non-null assertion would hide a removed column behind a confusing undefined access; this says + * which column went missing. + */ +function columnSpec(id: EvtxColumnId): EvtxColumnSpec { + const found = EVTX_COLUMNS.find((column) => column.id === id); + if (!found) throw new Error(`no column spec for ${id}`); + return found; +} + +const config = (order: string[], widths = {}): EvtxColumnConfig => + sanitizeColumnConfig({ order, widths }); + +function record(partial: Partial = {}): EvtxRecord { + return { + id: 0, + eventRecordId: 42, + timestamp: "2026-08-09 12:00:00", + timestampEpoch: 0, + provider: "ESENT", + channel: "Application", + eventId: 326, + level: "Error", + computer: "TESTHOST-01", + message: "something happened", + eventData: [], + rawXml: "", + sourceLabel: "Live", + ...partial, + }; +} + +describe("sanitizeColumnConfig", () => { + it("drops ids this build does not know", () => { + // Configuration outlives the build that wrote it; a removed column would render an empty cell. + expect(config(["level", "notAColumn", "provider"]).order).toEqual(["level", "provider"]); + }); + + it("deduplicates repeated ids", () => { + expect(config(["level", "level", "provider"]).order).toEqual(["level", "provider"]); + }); + + it("falls back to defaults when every column is hidden", () => { + // An empty list has no affordance to recover from. + expect(sanitizeColumnConfig({ order: [] }).order).toEqual(defaultColumnConfig().order); + expect(sanitizeColumnConfig(null).order).toEqual(defaultColumnConfig().order); + }); + + it("rejects widths that would hide a column the operator believes is shown", () => { + const sanitized = config(["level"], { level: 0, provider: -5, channel: 120 }); + expect(sanitized.widths.level).toBeUndefined(); + expect(sanitized.widths.provider).toBeUndefined(); + expect(sanitized.widths.channel).toBe(120); + }); + + it("ignores width entries for unknown columns", () => { + expect(config(["level"], { bogus: 100 }).widths).toEqual({}); + }); +}); + +describe("visibleColumns", () => { + it("returns specs in the configured order", () => { + expect(visibleColumns(config(["provider", "level"])).map((c) => c.id)).toEqual([ + "provider", + "level", + ]); + }); +}); + +describe("columnWidth", () => { + it("prefers an override over the default", () => { + const level = columnSpec("level"); + expect(columnWidth(config(["level"], { level: 88 }), level)).toBe(88); + expect(columnWidth(config(["level"]), level)).toBe(level.defaultWidth); + }); + + it("keeps the description column unbounded", () => { + const message = columnSpec("message"); + expect(columnWidth(config(["message"]), message)).toBeNull(); + }); +}); + +describe("moveColumn", () => { + it("swaps with its neighbour", () => { + const moved = moveColumn(config(["level", "provider", "channel"]), "provider", -1); + expect(moved.order).toEqual(["provider", "level", "channel"]); + }); + + it("ignores moves off either end", () => { + const start = config(["level", "provider"]); + expect(moveColumn(start, "level", -1).order).toEqual(start.order); + expect(moveColumn(start, "provider", 1).order).toEqual(start.order); + }); + + it("ignores a column that is not shown", () => { + const start = config(["level"]); + expect(moveColumn(start, "keywords", 1).order).toEqual(start.order); + }); +}); + +describe("toggleColumn", () => { + it("appends a newly shown column", () => { + expect(toggleColumn(config(["level"]), "keywords").order).toEqual(["level", "keywords"]); + }); + + it("removes a shown column", () => { + expect(toggleColumn(config(["level", "keywords"]), "keywords").order).toEqual(["level"]); + }); + + it("refuses to hide the last remaining column", () => { + const start = config(["level"]); + expect(toggleColumn(start, "level").order).toEqual(["level"]); + }); +}); + +describe("columnValue", () => { + it("renders present values", () => { + const r = record({ task: 13312, processId: 1234, keywords: "0x80" }); + expect(columnValue(r, "eventId")).toBe("326"); + expect(columnValue(r, "recordId")).toBe("42"); + expect(columnValue(r, "task")).toBe("13312"); + expect(columnValue(r, "processId")).toBe("1234"); + expect(columnValue(r, "keywords")).toBe("0x80"); + }); + + it("renders an absent value as empty rather than zero", () => { + // Consistent with the record model: 0 would be a value the provider never claimed. + const r = record({ task: undefined, opcode: null, threadId: null }); + expect(columnValue(r, "task")).toBe(""); + expect(columnValue(r, "opcode")).toBe(""); + expect(columnValue(r, "threadId")).toBe(""); + }); + + it("covers every declared column", () => { + const r = record(); + for (const column of EVTX_COLUMNS) { + expect(typeof columnValue(r, column.id)).toBe("string"); + } + }); +}); + +describe("the timestamp column", () => { + it("agrees with the day the same record groups under", () => { + // The bug this replaced: the column printed the raw UTC string Windows wrote while grouping + // bucketed by local date, so an evening event could show a UTC time from the following day + // while sitting under today's group. + const evening = Date.UTC(2026, 1, 10, 23, 30, 0); + const r = record({ timestampEpoch: evening, timestamp: "2026-02-10T23:30:00.000Z" }); + for (const zone of ["local", "utc"] as const) { + const shown = columnValue(r, "timestamp", zone); + expect(shown.slice(0, 10)).toBe(eventDateKey(evening, zone)); + } + }); + + it("shows UTC when UTC is selected", () => { + const r = record({ + timestampEpoch: Date.UTC(2026, 1, 10, 16, 36, 4, 390), + timestamp: "2026-02-10T16:36:04.390987Z", + }); + expect(columnValue(r, "timestamp", "utc")).toBe("2026-02-10 16:36:04.390987"); + }); + + it("defaults to local rather than to whatever the record string held", () => { + const epoch = Date.UTC(2026, 1, 10, 16, 36, 4, 390); + const r = record({ timestampEpoch: epoch, timestamp: "2026-02-10T16:36:04.390Z" }); + expect(columnValue(r, "timestamp")).toBe(formatEventTime(epoch, "local", r.timestamp)); + }); +}); + +describe("map columns", () => { + const mapped = (property: string, text: string, complete = true) => + record({ mapped: [{ property, text, complete }] }); + + it("renders a map-produced value", () => { + // The whole point of the map engine: these have to be scannable as a column, not reachable + // only by clicking each row open. + const r = mapped("PayloadData1", "cmd.exe"); + expect(columnValue(r, mappedColumnId("PayloadData1"))).toBe("cmd.exe"); + }); + + it("renders empty for an event the map did not match", () => { + expect(columnValue(record({}), mappedColumnId("PayloadData1"))).toBe(""); + expect(columnValue(mapped("UserName", "adam"), mappedColumnId("RemoteHost"))).toBe(""); + }); + + it("renders empty rather than showing an unsubstituted template", () => { + // A partially applied map would otherwise put a literal %3 in a column being scanned. + const r = mapped("PayloadData1", "ran %3 as adam", false); + expect(columnValue(r, mappedColumnId("PayloadData1"))).toBe(""); + }); + + it("offers only the properties the loaded records actually carry", () => { + // Offering everything every map could emit fills the chooser with columns that are empty for + // the log in front of the operator. + const properties = discoverMappedProperties([ + mapped("PayloadData1", "a"), + mapped("UserName", "b"), + mapped("PayloadData1", "c"), + record({}), + ]); + expect(properties).toEqual(["PayloadData1", "UserName"]); + }); + + it("lists fixed columns before map columns", () => { + const columns = availableColumns(["PayloadData1"]); + expect(columns.slice(0, EVTX_COLUMNS.length)).toEqual(EVTX_COLUMNS); + expect(columns[columns.length - 1]).toMatchObject({ + id: mappedColumnId("PayloadData1"), + label: "PayloadData1", + }); + }); + + it("keeps a stored map column even when its map is not loaded", () => { + // The maps loaded now need not be the ones loaded when the layout was saved. Dropping the + // column would silently discard an arrangement the operator made; an unmatched map column + // renders empty, exactly as an unmatched event already does. + const config = sanitizeColumnConfig({ + order: ["level", mappedColumnId("RemoteHost")], + widths: { [mappedColumnId("RemoteHost")]: 200 }, + }); + expect(config.order).toContain(mappedColumnId("RemoteHost")); + const visible = visibleColumns(config); + expect(visible.map((c) => c.id)).toEqual(["level", mappedColumnId("RemoteHost")]); + expect(columnWidth(config, visible[1])).toBe(200); + }); + + it("still rejects an id that is neither fixed nor a map column", () => { + const config = sanitizeColumnConfig({ order: ["level", "notAColumn", "mapped:"] }); + expect(config.order).toEqual(["level"]); + }); +}); diff --git a/src/workspaces/event-log/evtx-columns.ts b/src/workspaces/event-log/evtx-columns.ts new file mode 100644 index 000000000..bad6beef5 --- /dev/null +++ b/src/workspaces/event-log/evtx-columns.ts @@ -0,0 +1,309 @@ +/** + * Column configuration for the event list. + * + * FullEventLogView exposes sixteen columns with a chooser that reorders them and sets widths; we + * showed six with no control at all. The model is kept pure so ordering and validation are + * testable without a React or Tauri runtime. + */ +import type { EvtxRecord } from "./types"; +import { formatEventTime, type EvtxTimeZoneMode } from "./evtx-time"; + +/** + * A column whose meaning is fixed by the event schema. + */ +export type EvtxFixedColumnId = + | "level" + | "timestamp" + | "eventId" + | "recordId" + | "channel" + | "provider" + | "computer" + | "task" + | "opcode" + | "processId" + | "threadId" + | "keywords" + | "message"; + +/** + * A column produced by an event map, such as `PayloadData1` or `RemoteHost`. + * + * These cannot be a fixed list: which ones exist depends on which maps are loaded and which events + * they matched. Encoding the property in the id keeps the configuration a flat list of strings, + * which is what makes it survive being written to disk by one build and read by another. + */ +export type EvtxMappedColumnId = `mapped:${string}`; + +export type EvtxColumnId = EvtxFixedColumnId | EvtxMappedColumnId; + +const MAPPED_PREFIX = "mapped:"; + +/** The column id carrying an event map's `property`. */ +export function mappedColumnId(property: string): EvtxMappedColumnId { + return `${MAPPED_PREFIX}${property}`; +} + +/** The map property behind a column id, or null when the column is a fixed one. */ +export function mappedColumnProperty(id: string): string | null { + return id.startsWith(MAPPED_PREFIX) ? id.slice(MAPPED_PREFIX.length) : null; +} + +/** + * The map columns present in a set of records. + * + * Discovered from the data rather than declared, because a map only contributes a column to events + * it actually matched. Offering every property every map could ever emit would fill the chooser + * with columns that are empty for the log in front of the operator. + */ +export function discoverMappedProperties( + records: readonly EvtxRecord[] +): string[] { + const seen = new Set(); + for (const record of records) { + for (const column of record.mapped ?? []) { + seen.add(column.property); + } + } + return [...seen].sort(); +} + +/** A renderable spec for a map column, derived from its id alone. */ +function mappedColumnSpec(id: EvtxMappedColumnId): EvtxColumnSpec { + return { + id, + label: mappedColumnProperty(id) ?? id, + defaultWidth: 140, + defaultVisible: false, + }; +} + +/** + * Every column offerable for the loaded records: the fixed ones, then whatever the maps produced. + */ +export function availableColumns( + mappedProperties: readonly string[] +): EvtxColumnSpec[] { + return [ + ...EVTX_COLUMNS, + ...mappedProperties.map((property) => mappedColumnSpec(mappedColumnId(property))), + ]; +} + +export interface EvtxColumnSpec { + id: EvtxColumnId; + label: string; + /** Default width in pixels, or null for the column that absorbs remaining space. */ + defaultWidth: number | null; + /** Shown when no configuration has been saved. */ + defaultVisible: boolean; +} + +/** Every column, in the order a fresh install presents them. */ +export const EVTX_COLUMNS: EvtxColumnSpec[] = [ + { id: "level", label: "Level", defaultWidth: 40, defaultVisible: true }, + { id: "timestamp", label: "Event Time", defaultWidth: 165, defaultVisible: true }, + { id: "eventId", label: "Event ID", defaultWidth: 50, defaultVisible: true }, + { id: "recordId", label: "Record ID", defaultWidth: 70, defaultVisible: false }, + { id: "channel", label: "Channel", defaultWidth: 140, defaultVisible: true }, + { id: "provider", label: "Provider", defaultWidth: 160, defaultVisible: true }, + { id: "computer", label: "Computer", defaultWidth: 120, defaultVisible: false }, + { id: "task", label: "Task", defaultWidth: 60, defaultVisible: false }, + { id: "opcode", label: "Opcode", defaultWidth: 60, defaultVisible: false }, + { id: "processId", label: "PID", defaultWidth: 60, defaultVisible: false }, + { id: "threadId", label: "TID", defaultWidth: 60, defaultVisible: false }, + { id: "keywords", label: "Keywords", defaultWidth: 140, defaultVisible: false }, + // Last and unbounded: the description absorbs whatever width remains. + { id: "message", label: "Description", defaultWidth: null, defaultVisible: true }, +]; + +const COLUMN_IDS = new Set(EVTX_COLUMNS.map((column) => column.id)); + +/** + * Fixed columns keyed by id. + * + * Module scope because EVTX_COLUMNS never changes and the row renderer calls visibleColumns once + * per rendered row; rebuilding the map there allocated one per row per render. + */ +const FIXED_COLUMNS_BY_ID = new Map( + EVTX_COLUMNS.map((column) => [column.id, column]) +); + +/** + * Whether a stored id is one this build can render. + * + * A map column is accepted whatever its property, because the maps loaded at the time the + * configuration was written may not be loaded now. Dropping it would silently discard a column the + * operator arranged, and it costs nothing to keep: a map column with no matching value renders + * empty, exactly as an event that the map did not match already does. + */ +function isKnownColumnId(candidate: string): boolean { + if (COLUMN_IDS.has(candidate)) return true; + const property = mappedColumnProperty(candidate); + return property !== null && property.length > 0; +} + +export interface EvtxColumnConfig { + /** Visible columns in display order. */ + order: EvtxColumnId[]; + /** Width overrides, keyed by column id. */ + widths: Partial>; +} + +/** The configuration a fresh install starts from. */ +export function defaultColumnConfig(): EvtxColumnConfig { + return { + order: EVTX_COLUMNS.filter((column) => column.defaultVisible).map((column) => column.id), + widths: {}, + }; +} + +/** + * Coerces stored configuration into something this build can render. + * + * Configuration outlives the build that wrote it. A removed column would otherwise render as an + * empty cell forever, and a column added by a later build would be invisible with no way to reach + * it, so unknown ids are dropped and the order is deduplicated. + */ +export function sanitizeColumnConfig(input: unknown): EvtxColumnConfig { + const raw = + typeof input === "object" && input !== null && !Array.isArray(input) + ? (input as Record) + : {}; + + const seen = new Set(); + const order: EvtxColumnId[] = []; + if (Array.isArray(raw.order)) { + for (const candidate of raw.order) { + if (typeof candidate !== "string" || !isKnownColumnId(candidate)) continue; + const id = candidate as EvtxColumnId; + if (seen.has(id)) continue; + seen.add(id); + order.push(id); + } + } + + const widths: Partial> = {}; + if (typeof raw.widths === "object" && raw.widths !== null) { + for (const [key, value] of Object.entries(raw.widths as Record)) { + if (!isKnownColumnId(key)) continue; + // A zero or negative width would hide a column the operator believes is shown. + if (typeof value === "number" && Number.isFinite(value) && value >= 24) { + widths[key as EvtxColumnId] = Math.round(value); + } + } + } + + // Every column hidden leaves an empty list with no way back, so that falls back to the defaults. + return order.length > 0 ? { order, widths } : { ...defaultColumnConfig(), widths }; +} + +/** Specs for the visible columns, in display order. */ +export function visibleColumns(config: EvtxColumnConfig): EvtxColumnSpec[] { + return config.order + .map((id) => { + const fixed = FIXED_COLUMNS_BY_ID.get(id); + if (fixed) return fixed; + // Synthesized from the id, so rendering a map column needs no knowledge of which maps are + // loaded. That keeps the row renderer independent of load order. + return mappedColumnProperty(id) ? mappedColumnSpec(id as EvtxMappedColumnId) : undefined; + }) + .filter((column): column is EvtxColumnSpec => column !== undefined); +} + +/** Width to render `column` at, honouring any override. */ +export function columnWidth(config: EvtxColumnConfig, column: EvtxColumnSpec): number | null { + const override = config.widths[column.id]; + if (override !== undefined) return override; + return column.defaultWidth; +} + +/** Moves a column one position earlier or later, ignoring moves off either end. */ +export function moveColumn( + config: EvtxColumnConfig, + id: EvtxColumnId, + direction: -1 | 1 +): EvtxColumnConfig { + const index = config.order.indexOf(id); + const target = index + direction; + if (index < 0 || target < 0 || target >= config.order.length) return config; + const order = [...config.order]; + [order[index], order[target]] = [order[target], order[index]]; + return { ...config, order }; +} + +/** Shows or hides a column, appending a newly shown one at the end. */ +export function toggleColumn(config: EvtxColumnConfig, id: EvtxColumnId): EvtxColumnConfig { + if (config.order.includes(id)) { + const order = config.order.filter((existing) => existing !== id); + // Refuse to hide the last column; an empty list has no affordance to recover from. + return order.length > 0 ? { ...config, order } : config; + } + return { ...config, order: [...config.order, id] }; +} + +/** + * Renders a record's value for a column, as displayed text. + * + * The time zone is explicit rather than implied. This column previously printed the raw string + * Windows wrote, which is UTC, while the rest of the workspace showed local time, so the same + * event carried two different clocks depending on where it was read. + */ +export function columnValue( + record: EvtxRecord, + id: EvtxColumnId, + timeZone: EvtxTimeZoneMode = "local" +): string { + const mappedProperty = mappedColumnProperty(id); + if (mappedProperty !== null) { + const column = record.mapped?.find((entry) => entry.property === mappedProperty); + // An event the map did not match, or matched incompletely, renders empty. Showing a partially + // substituted template would put a literal %3 in a column an operator is scanning. + return column && column.complete ? column.text : ""; + } + return fixedColumnValue(record, id as EvtxFixedColumnId, timeZone); +} + +/** + * Renders a fixed column. + * + * Split out so the switch stays exhaustive over `EvtxFixedColumnId`. Folding map columns into the + * same switch would widen it to a template literal type and lose the compile error that catches a + * newly added column nobody wrote a case for. + */ +function fixedColumnValue( + record: EvtxRecord, + id: EvtxFixedColumnId, + timeZone: EvtxTimeZoneMode +): string { + switch (id) { + case "level": + return record.level; + case "timestamp": + return formatEventTime(record.timestampEpoch, timeZone, record.timestamp); + case "eventId": + return String(record.eventId); + case "recordId": + return String(record.eventRecordId); + case "channel": + return record.channel; + case "provider": + return record.provider; + case "computer": + return record.computer; + // Absent values render empty rather than as 0, matching the record model: the provider wrote + // nothing, and 0 would be a value it never claimed. + case "task": + return record.task != null ? String(record.task) : ""; + case "opcode": + return record.opcode != null ? String(record.opcode) : ""; + case "processId": + return record.processId != null ? String(record.processId) : ""; + case "threadId": + return record.threadId != null ? String(record.threadId) : ""; + case "keywords": + return record.keywords ?? ""; + case "message": + return record.message; + } +} diff --git a/src/workspaces/event-log/evtx-coverage.test.ts b/src/workspaces/event-log/evtx-coverage.test.ts new file mode 100644 index 000000000..4db654f86 --- /dev/null +++ b/src/workspaces/event-log/evtx-coverage.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from "vitest"; +import { + assertParseResultShape, + mergeCoverageGaps, + summarizeCoverageGaps, +} from "./evtx-coverage"; + +describe("mergeCoverageGaps", () => { + it("accumulates gaps across channels", () => { + // Each channel loads separately and reports its own gaps. Replacing rather than accumulating + // would leave only the last channel's gaps visible and silently drop the rest. + const merged = mergeCoverageGaps(["Application: 3 records unreadable"], [ + "System: stopped at 100000 events", + ]); + expect(merged).toEqual([ + "Application: 3 records unreadable", + "System: stopped at 100000 events", + ]); + }); + + it("does not repeat a gap when a channel is re-queried", () => { + // A banner that grows on every refresh trains an operator to stop reading it. + const first = mergeCoverageGaps([], ["Application: 3 records unreadable"]); + const second = mergeCoverageGaps(first, ["Application: 3 records unreadable"]); + expect(second).toHaveLength(1); + }); + + it("keeps the order gaps were first reported in", () => { + // A gap that moves around the list as later channels finish is hard to read past. + const merged = mergeCoverageGaps(["first", "second"], ["third", "first"]); + expect(merged).toEqual(["first", "second", "third"]); + }); + + it("reports nothing when nothing is missing", () => { + expect(mergeCoverageGaps([], [])).toEqual([]); + }); +}); + +describe("summarizeCoverageGaps", () => { + it("uses the singular for one gap", () => { + expect(summarizeCoverageGaps(["only"])).toBe("1 gap in this view"); + }); + + it("uses the plural otherwise", () => { + expect(summarizeCoverageGaps(["a", "b"])).toBe("2 gaps in this view"); + }); +}); + +describe("gaps across load paths", () => { + it("a refresh replaces gaps rather than carrying stale ones forward", () => { + // The refresh clears records, so the gaps describing them have to go too. Keeping them would + // report a gap from a set no longer on screen while the new result's own gap went unsaid. + const beforeRefresh = ["Application: 3 records unreadable"]; + const afterClear = mergeCoverageGaps([], ["System: stopped at 100000 events"]); + expect(afterClear).not.toContain(beforeRefresh[0]); + expect(afterClear).toEqual(["System: stopped at 100000 events"]); + }); + + it("an incremental channel load adds to what is already reported", () => { + // Channels load one at a time, so replacing here would leave only the last channel's gaps. + const first = mergeCoverageGaps([], ["Application: 3 records unreadable"]); + const second = mergeCoverageGaps(first, ["Security: 1 record unreadable"]); + expect(second).toHaveLength(2); + }); +}); + +describe("assertParseResultShape", () => { + it("accepts a well formed reply", () => { + const shape = assertParseResultShape({ + records: [], + channels: [], + errorMessages: ["Application: 3 records unreadable"], + }); + expect(shape.errorMessages).toEqual(["Application: 3 records unreadable"]); + }); + + it("treats absent errorMessages as no gaps rather than a malformed reply", () => { + // An older reader that reports nothing is not the same as one this build cannot read. + expect(assertParseResultShape({ records: [], channels: [] }).errorMessages).toEqual([]); + }); + + it("rejects a reply whose records are not a list", () => { + // Spreading this would throw somewhere unrelated and surface as a confusing load error. + expect(() => assertParseResultShape({ records: null, channels: [] })).toThrow( + /cannot read/ + ); + expect(() => assertParseResultShape({ channels: [] })).toThrow(/cannot read/); + expect(() => assertParseResultShape(undefined)).toThrow(/cannot read/); + }); + + it("drops non-string gap entries rather than rendering them", () => { + const shape = assertParseResultShape({ + records: [], + channels: [], + errorMessages: ["real", 42, null], + }); + expect(shape.errorMessages).toEqual(["real"]); + }); +}); diff --git a/src/workspaces/event-log/evtx-coverage.ts b/src/workspaces/event-log/evtx-coverage.ts new file mode 100644 index 000000000..e049c390c --- /dev/null +++ b/src/workspaces/event-log/evtx-coverage.ts @@ -0,0 +1,75 @@ +/** + * Accumulating the report of what is missing from a loaded set of events. + * + * Lives outside the store because the store imports the Tauri IPC bridge at module scope, which a + * unit test cannot load. The rule being tested is small but easy to get wrong in a way nobody + * notices: gaps must accumulate across channels, and must not multiply when a channel is + * re-queried. + */ + +/** + * Merges newly reported gaps into the ones already on screen. + * + * Channels load one at a time and each reports its own gaps, so these accumulate rather than + * replace. They are deduplicated because re-querying a channel reports the same gap again, and a + * banner that grows every refresh trains an operator to stop reading it. + * + * Order is preserved so a gap does not move around the list as more channels finish. + */ +export function mergeCoverageGaps( + existing: readonly string[], + incoming: readonly string[] +): string[] { + return [...new Set([...existing, ...incoming])]; +} + +/** Wording for the banner summary. */ +export function summarizeCoverageGaps(gaps: readonly string[]): string { + return gaps.length === 1 ? "1 gap in this view" : `${gaps.length} gaps in this view`; +} + +/** + * The parts of an event-log IPC reply the store reads, verified once. + * + * Not a schema validator, and deliberately not per-handler checks: it guards the three fields the + * store destructures and iterates. If a future backend change dropped `errorMessages`, spreading + * it would throw somewhere unrelated and surface as a confusing load error; this fails at the + * boundary with a message that names the contract. + * + * Throws rather than returning a default, because a reply the store cannot read is not a reply + * with no events, and quietly showing an empty list is the failure this workspace exists to avoid. + */ +export function assertParseResultShape(value: unknown): { + records: unknown[]; + channels: unknown[]; + errorMessages: string[]; + totalRecords: number | null; +} { + const reply = value as { + records?: unknown; + channels?: unknown; + errorMessages?: unknown; + totalRecords?: unknown; + }; + if (!Array.isArray(reply?.records) || !Array.isArray(reply?.channels)) { + throw new Error("the event log reader returned a reply this build cannot read"); + } + return { + records: reply.records, + channels: reply.channels, + // Absent means the reader reported no gaps, which is different from a malformed reply. + errorMessages: Array.isArray(reply.errorMessages) + ? reply.errorMessages.filter((entry): entry is string => typeof entry === "string") + : [], + // How many records the reader says it sent, counting any streamed separately from this reply. + // `null` when the reader did not say, which must stay distinguishable from zero: treating an + // absent count as zero would turn "I cannot check completeness" into "nothing was missing". + // A non-finite or negative count is no answer either, so it is rejected the same way. + totalRecords: + typeof reply.totalRecords === "number" && + Number.isFinite(reply.totalRecords) && + reply.totalRecords >= 0 + ? reply.totalRecords + : null, + }; +} diff --git a/src/workspaces/event-log/evtx-filter-store.test.ts b/src/workspaces/event-log/evtx-filter-store.test.ts new file mode 100644 index 000000000..a77ab6424 --- /dev/null +++ b/src/workspaces/event-log/evtx-filter-store.test.ts @@ -0,0 +1,86 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { useSavedFilterStore } from "./evtx-filter-store"; +import { sanitizeCriteria } from "./evtx-saved-filters"; + +const criteria = () => sanitizeCriteria({ levels: ["Error"], search: "boot" }); + +/** save() returns null for an empty name; every call here uses a real one, so assert that. */ +function saveNamed(name: string) { + const saved = useSavedFilterStore.getState().save(name, criteria()); + if (!saved) throw new Error(`save refused the name ${JSON.stringify(name)}`); + return saved; +} + +beforeEach(() => { + useSavedFilterStore.setState({ savedFilters: [] }); + localStorage.clear(); +}); + +describe("useSavedFilterStore", () => { + it("saves a filter and stamps it as used", () => { + const saved = saveNamed("Boot errors"); + expect(saved.name).toBe("Boot errors"); + expect(saved.lastUsed).not.toBeNull(); + expect(useSavedFilterStore.getState().savedFilters).toHaveLength(1); + }); + + it("saving under an existing name updates rather than duplicating", () => { + const first = saveNamed("Boot"); + const second = useSavedFilterStore + .getState() + .save("boot", sanitizeCriteria({ search: "changed" })); + if (!second) throw new Error("save refused a valid name"); + + expect(useSavedFilterStore.getState().savedFilters).toHaveLength(1); + expect(second.id).toBe(first.id); + expect(useSavedFilterStore.getState().savedFilters[0].criteria.search).toBe("changed"); + }); + + it("preserves the favorite flag when re-saving", () => { + const saved = saveNamed("Boot"); + useSavedFilterStore.getState().toggleFavorite(saved.id); + useSavedFilterStore.getState().save("Boot", sanitizeCriteria({ search: "again" })); + expect(useSavedFilterStore.getState().savedFilters[0].favorite).toBe(true); + }); + + it("removes by id", () => { + const saved = saveNamed("Boot"); + useSavedFilterStore.getState().remove(saved.id); + expect(useSavedFilterStore.getState().savedFilters).toEqual([]); + }); + + it("orders favorites first", () => { + // Favourite the one that loses on every other rule. Favouriting "Alpha" proved nothing: both + // saves stamp lastUsed from the same clock tick, so the ordering fell through to the name + // comparison and "Alpha" came first whether or not toggleFavorite did anything at all. + const zulu = saveNamed("Zulu"); + saveNamed("Alpha"); + + expect(useSavedFilterStore.getState().ordered()[0].name).toBe("Alpha"); + + useSavedFilterStore.getState().toggleFavorite(zulu.id); + expect(useSavedFilterStore.getState().ordered()[0].name).toBe("Zulu"); + }); + + it("drops persisted entries that no longer validate rather than repairing them", () => { + // Persisted data outlives the build that wrote it. Repairing a filter into something the + // operator never chose would be worse than losing it. + const merged = ( + useSavedFilterStore.persist.getOptions().merge as ( + persisted: unknown, + current: unknown + ) => { savedFilters: unknown[] } + )( + { savedFilters: [{ name: "Good" }, { noName: true }, "nonsense"] }, + { savedFilters: [] } + ); + expect(merged.savedFilters).toHaveLength(1); + }); + + it("refuses a whitespace-only name instead of storing one that vanishes", () => { + // sanitizeSavedFilter drops an empty name on rehydration, so storing it would show the filter + // in the list and then lose it on restart, which reads as the app losing the operator's work. + expect(useSavedFilterStore.getState().save(" ", criteria())).toBeNull(); + expect(useSavedFilterStore.getState().savedFilters).toHaveLength(0); + }); +}); diff --git a/src/workspaces/event-log/evtx-filter-store.ts b/src/workspaces/event-log/evtx-filter-store.ts new file mode 100644 index 000000000..2cca27505 --- /dev/null +++ b/src/workspaces/event-log/evtx-filter-store.ts @@ -0,0 +1,103 @@ +/** + * Persisted store for the saved filter library. + * + * Deliberately separate from `evtx-store`, which holds loaded records. Those are large, derived, + * and machine-specific, so persisting that store would write megabytes of event data into local + * storage. This one holds only the criteria an operator chose to keep. + */ +import { create } from "zustand"; +import { persist } from "zustand/middleware"; + +import { + mergeFilters, + orderFilters, + sanitizeSavedFilter, + type EvtxFilterCriteria, + type EvtxSavedFilter, +} from "./evtx-saved-filters"; + +interface SavedFilterState { + savedFilters: EvtxSavedFilter[]; + /** Returns the stored filter, or null when the name is empty once trimmed. */ + save: (name: string, criteria: EvtxFilterCriteria) => EvtxSavedFilter | null; + remove: (id: string) => void; + toggleFavorite: (id: string) => void; + markUsed: (id: string) => void; + importFilters: (imported: EvtxSavedFilter[]) => void; + ordered: () => EvtxSavedFilter[]; +} + +function newId(): string { + // crypto.randomUUID is unavailable in some webviews, so fall back rather than throwing. + if (typeof crypto !== "undefined" && "randomUUID" in crypto) { + return crypto.randomUUID(); + } + return `filter-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; +} + +export const useSavedFilterStore = create()( + persist( + (set, get) => ({ + savedFilters: [], + + save: (name, criteria) => { + const trimmed = name.trim(); + // A whitespace-only name is refused rather than stored. sanitizeSavedFilter drops it on + // rehydration, so it would save, appear in the list, and then vanish on restart, which + // reads as the app losing the operator's filter. + if (!trimmed) return null; + const existing = get().savedFilters.find( + (filter) => filter.name.toLowerCase() === trimmed.toLowerCase() + ); + // Saving under an existing name updates it rather than creating a second entry, which is + // what "save" means when the name already exists in the list the operator is looking at. + const filter: EvtxSavedFilter = { + id: existing?.id ?? newId(), + name: trimmed, + favorite: existing?.favorite ?? false, + tags: existing?.tags ?? [], + criteria, + lastUsed: Date.now(), + }; + set({ savedFilters: mergeFilters(get().savedFilters, [filter]) }); + return filter; + }, + + remove: (id) => + set({ savedFilters: get().savedFilters.filter((filter) => filter.id !== id) }), + + toggleFavorite: (id) => + set({ + savedFilters: get().savedFilters.map((filter) => + filter.id === id ? { ...filter, favorite: !filter.favorite } : filter + ), + }), + + markUsed: (id) => + set({ + savedFilters: get().savedFilters.map((filter) => + filter.id === id ? { ...filter, lastUsed: Date.now() } : filter + ), + }), + + importFilters: (imported) => + set({ savedFilters: mergeFilters(get().savedFilters, imported) }), + + ordered: () => orderFilters(get().savedFilters), + }), + { + name: "cmtraceopen-evtx-saved-filters", + // Persisted data outlives the build that wrote it, so it is revalidated on load rather than + // trusted. A filter that no longer validates is dropped, not repaired into something the + // operator never chose. + merge: (persisted, current) => { + const raw = persisted as { savedFilters?: unknown } | undefined; + const list = Array.isArray(raw?.savedFilters) ? raw.savedFilters : []; + const savedFilters = list + .map((entry, index) => sanitizeSavedFilter(entry, `restored-${index}`)) + .filter((filter): filter is EvtxSavedFilter => filter !== null); + return { ...current, savedFilters }; + }, + } + ) +); diff --git a/src/workspaces/event-log/evtx-filter.test.ts b/src/workspaces/event-log/evtx-filter.test.ts new file mode 100644 index 000000000..e03b28fdf --- /dev/null +++ b/src/workspaces/event-log/evtx-filter.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from "vitest"; +import { parseEventIdFilter } from "./evtx-filter"; + +describe("parseEventIdFilter", () => { + it("returns null when the box constrains nothing", () => { + expect(parseEventIdFilter("")).toBeNull(); + expect(parseEventIdFilter(" ")).toBeNull(); + }); + + it("parses a comma separated list", () => { + expect(parseEventIdFilter("4624,4625")).toEqual(new Set([4624, 4625])); + }); + + it("tolerates spaces as separators and around commas", () => { + expect(parseEventIdFilter("4624 4625")).toEqual(new Set([4624, 4625])); + expect(parseEventIdFilter(" 4624 , 4625 ")).toEqual(new Set([4624, 4625])); + }); + + it("expands an inclusive range", () => { + expect(parseEventIdFilter("5-8")).toEqual(new Set([5, 6, 7, 8])); + }); + + it("normalizes a reversed range rather than yielding nothing", () => { + expect(parseEventIdFilter("8-5")).toEqual(new Set([5, 6, 7, 8])); + }); + + it("mixes singles and ranges", () => { + expect(parseEventIdFilter("1, 4-6, 9")).toEqual(new Set([1, 4, 5, 6, 9])); + }); + + it("ignores tokens that are not ids instead of failing the whole filter", () => { + // A half-typed filter should narrow by what is parseable, not silently match everything. + expect(parseEventIdFilter("4624, abc")).toEqual(new Set([4624])); + }); + + it("returns null when nothing in the box parsed", () => { + expect(parseEventIdFilter("abc, def")).toBeNull(); + }); +}); + +describe("event id range bounds", () => { + it("does not expand past the 16-bit event id space", () => { + // Typed on every keystroke. Unbounded, "4624-46240000" builds a set of tens of millions on the + // UI thread and the tab stops responding before the operator finishes typing. + const started = Date.now(); + const ids = parseEventIdFilter("4624-46240000"); + expect(Date.now() - started).toBeLessThan(1000); + expect(ids).not.toBeNull(); + expect(ids!.size).toBeLessThanOrEqual(65536); + expect(ids!.has(4624)).toBe(true); + expect(ids!.has(65535)).toBe(true); + expect(ids!.has(65536)).toBe(false); + }); + + it("yields nothing for a range entirely above the id space", () => { + expect(parseEventIdFilter("100000-200000")).toBeNull(); + }); + + it("still expands an ordinary range", () => { + const ids = parseEventIdFilter("4624-4626"); + expect([...ids!].sort((a, b) => a - b)).toEqual([4624, 4625, 4626]); + }); +}); diff --git a/src/workspaces/event-log/evtx-filter.ts b/src/workspaces/event-log/evtx-filter.ts new file mode 100644 index 000000000..c962ecd5c --- /dev/null +++ b/src/workspaces/event-log/evtx-filter.ts @@ -0,0 +1,212 @@ +/** + * Pure filtering helpers for the event log view. + * + * Deliberately separate from `evtx-store.ts`, which subscribes to Tauri events at module scope. + * Importing the store from a test fires that subscription, so anything worth unit-testing lives + * here where it can be imported without a Tauri runtime. + */ +import type { EvtxLevel, EvtxRecord } from "./types"; +import { eventDateKey, type EvtxTimeZoneMode } from "./evtx-time"; + +/** + * Parses the Event ID filter box into a set, or null when it constrains nothing. + * + * Accepts comma or space separated ids and inclusive `low-high` ranges, matching what operators + * expect from the incumbent tools. + */ +export function parseEventIdFilter(raw: string): Set | null { + const trimmed = raw.trim(); + if (!trimmed) return null; + const ids = new Set(); + for (const token of trimmed.split(/[\s,]+/).filter(Boolean)) { + const range = token.match(/^(\d+)-(\d+)$/); + if (range) { + const low = Number(range[1]); + const high = Number(range[2]); + const [from, to] = low <= high ? [low, high] : [high, low]; + // Clamped to the range an Event ID can actually occupy. This runs on every keystroke, so + // "4624-46240000" typed halfway would otherwise build a set of tens of millions on the UI + // thread and freeze the tab before the operator finished the number. + if (from > MAX_EVENT_ID) continue; + for (let id = from; id <= Math.min(to, MAX_EVENT_ID); id += 1) ids.add(id); + continue; + } + const single = Number(token); + if (Number.isInteger(single)) ids.add(single); + } + return ids.size > 0 ? ids : null; +} + +/** + * Largest value a Windows Event ID can hold. + * + * The field is 16 bits, so nothing above this can match an event and expanding past it only costs + * time. Used to bound range expansion rather than to reject input: an operator mid-way through + * typing a number should see no result, not an error. + */ +const MAX_EVENT_ID = 65535; + +/** The subset of store state that decides which records are on screen. */ +export interface VisibleRecordsInput { + records: EvtxRecord[]; + selectedChannels: Set; + filterLevels: Set; + filterEventIds: string; + filterSearch: string; +} + +/** + * The records currently on screen, before sorting. + * + * Shared so an export writes exactly what the operator is looking at. Recomputing the predicate at + * the export site would let the two drift, and an export that quietly differs from the view is + * worse than no export at all. + */ +export function selectVisibleRecords(input: VisibleRecordsInput): EvtxRecord[] { + const eventIdSet = parseEventIdFilter(input.filterEventIds); + const search = input.filterSearch.trim().toLowerCase(); + return input.records.filter((r) => { + if (!input.selectedChannels.has(r.channel)) return false; + if (!input.filterLevels.has(r.level)) return false; + if (eventIdSet && !eventIdSet.has(r.eventId)) return false; + if ( + search && + !r.message.toLowerCase().includes(search) && + !r.provider.toLowerCase().includes(search) + ) { + return false; + } + return true; + }); +} + +/** A field the event list can group by. */ +export type EvtxGroupField = "level" | "provider" | "channel" | "eventId" | "day"; + +export const EVTX_GROUP_LABELS: Record = { + level: "Level", + provider: "Provider", + channel: "Channel", + eventId: "Event ID", + day: "Day", +}; + +/** A group header row. */ +export interface EvtxGroupRow { + kind: "group"; + /** Stable identity across renders, built from the whole ancestry so sibling groups never collide. */ + key: string; + field: EvtxGroupField; + label: string; + /** Nesting level, zero for the outermost grouping. */ + depth: number; + /** Records beneath this header, including those inside nested groups. */ + count: number; + collapsed: boolean; +} + +/** A record row. */ +export interface EvtxRecordRow { + kind: "record"; + record: EvtxRecord; + depth: number; +} + +export type EvtxRow = EvtxGroupRow | EvtxRecordRow; + +function groupValue( + record: EvtxRecord, + field: EvtxGroupField, + timeZone: EvtxTimeZoneMode +): string { + switch (field) { + case "level": + return record.level; + case "provider": + return record.provider || "(no provider)"; + case "channel": + return record.channel || "(no channel)"; + case "eventId": + return String(record.eventId); + case "day": + // The same zone the timestamps are displayed in, so an event never appears under a day that + // disagrees with the time printed beside it. + return eventDateKey(record.timestampEpoch, timeZone); + } +} + +/** + * Flattens records into the row list a virtualized list renders. + * + * Returns a flat array rather than a tree because the list virtualizes on row index; a tree would + * have to be flattened on every render anyway. + * + * Group order follows `groupBy`, and within the deepest group the incoming record order is + * preserved, so whatever sort the operator chose still applies inside each group. + */ +export function buildGroupedRows( + records: EvtxRecord[], + groupBy: EvtxGroupField[], + collapsedKeys: ReadonlySet, + timeZone: EvtxTimeZoneMode = "local" +): EvtxRow[] { + if (groupBy.length === 0) { + return records.map((record) => ({ kind: "record", record, depth: 0 })); + } + + const rows: EvtxRow[] = []; + + const walk = (subset: EvtxRecord[], depth: number, parentKey: string) => { + if (depth >= groupBy.length) { + for (const record of subset) { + rows.push({ kind: "record", record, depth }); + } + return; + } + + const field = groupBy[depth]; + // Insertion order is preserved by Map, so groups appear in the order they are first seen, + // which follows the caller's sort rather than imposing an alphabetical one. + const buckets = new Map(); + for (const record of subset) { + const value = groupValue(record, field, timeZone); + const bucket = buckets.get(value); + if (bucket) bucket.push(record); + else buckets.set(value, [record]); + } + + for (const [value, bucket] of buckets) { + // Encoded, so a value containing the delimiters cannot forge another group's ancestry. Two + // distinct paths sharing a key would make collapsing one collapse the other. + const key = `${parentKey}/${field}=${encodeURIComponent(value)}`; + const collapsed = collapsedKeys.has(key); + rows.push({ + kind: "group", + key, + field, + label: value, + depth, + count: bucket.length, + collapsed, + }); + if (!collapsed) { + walk(bucket, depth + 1, key); + } + } + }; + + walk(records, 0, ""); + return rows; +} + +/** Every group key present for `records` under `groupBy`, for expand-all and collapse-all. */ +export function allGroupKeys( + records: EvtxRecord[], + groupBy: EvtxGroupField[] +): Set { + const keys = new Set(); + for (const row of buildGroupedRows(records, groupBy, new Set())) { + if (row.kind === "group") keys.add(row.key); + } + return keys; +} diff --git a/src/workspaces/event-log/evtx-grouping.test.ts b/src/workspaces/event-log/evtx-grouping.test.ts new file mode 100644 index 000000000..1c7ae9ed4 --- /dev/null +++ b/src/workspaces/event-log/evtx-grouping.test.ts @@ -0,0 +1,239 @@ +import { describe, expect, it } from "vitest"; +import { + allGroupKeys, + buildGroupedRows, + type EvtxGroupField, + type EvtxRow, +} from "./evtx-filter"; +import type { EvtxRecord } from "./types"; + +function record(partial: Partial): EvtxRecord { + return { + id: 0, + eventRecordId: 0, + timestamp: "", + timestampEpoch: 0, + provider: "P", + channel: "C", + eventId: 1, + level: "Information", + computer: "H", + message: "", + eventData: [], + rawXml: "", + sourceLabel: "Live", + ...partial, + }; +} + +const groups = (rows: EvtxRow[]) => + rows.filter((r): r is Extract => r.kind === "group"); +const records = (rows: EvtxRow[]) => rows.filter((r) => r.kind === "record"); + +describe("buildGroupedRows", () => { + it("returns plain record rows when nothing is grouped", () => { + const rows = buildGroupedRows([record({ id: 1 }), record({ id: 2 })], [], new Set()); + expect(rows).toHaveLength(2); + expect(rows.every((r) => r.kind === "record")).toBe(true); + }); + + it("emits a header per distinct value with its record count", () => { + const rows = buildGroupedRows( + [ + record({ id: 1, level: "Error" }), + record({ id: 2, level: "Error" }), + record({ id: 3, level: "Warning" }), + ], + ["level"], + new Set() + ); + const headers = groups(rows); + expect(headers.map((h) => [h.label, h.count])).toEqual([ + ["Error", 2], + ["Warning", 1], + ]); + expect(records(rows)).toHaveLength(3); + }); + + it("preserves incoming order so the operator's sort still applies", () => { + // Warning is seen first, so its group comes first. Alphabetising would override the sort. + const rows = buildGroupedRows( + [record({ id: 1, level: "Warning" }), record({ id: 2, level: "Error" })], + ["level"], + new Set() + ); + expect(groups(rows).map((h) => h.label)).toEqual(["Warning", "Error"]); + }); + + it("nests groups in the order given and counts every descendant", () => { + const rows = buildGroupedRows( + [ + record({ id: 1, level: "Error", provider: "A" }), + record({ id: 2, level: "Error", provider: "B" }), + record({ id: 3, level: "Warning", provider: "A" }), + ], + ["level", "provider"], + new Set() + ); + const headers = groups(rows); + expect(headers.map((h) => [h.depth, h.label, h.count])).toEqual([ + [0, "Error", 2], + [1, "A", 1], + [1, "B", 1], + [0, "Warning", 1], + [1, "A", 1], + ]); + }); + + it("gives sibling groups distinct keys even when their labels match", () => { + // Provider A appears under both levels; colliding keys would collapse both at once. + const rows = buildGroupedRows( + [ + record({ id: 1, level: "Error", provider: "A" }), + record({ id: 2, level: "Warning", provider: "A" }), + ], + ["level", "provider"], + new Set() + ); + const keys = groups(rows) + .filter((h) => h.depth === 1) + .map((h) => h.key); + expect(new Set(keys).size).toBe(2); + }); + + it("hides descendants of a collapsed group but keeps its count", () => { + const rows = buildGroupedRows( + [ + record({ id: 1, level: "Error", provider: "A" }), + record({ id: 2, level: "Error", provider: "B" }), + record({ id: 3, level: "Warning" }), + ], + ["level", "provider"], + new Set(["/level=Error"]) + ); + const headers = groups(rows); + expect(headers.find((h) => h.label === "Error")?.count).toBe(2); + expect(headers.some((h) => h.depth === 1 && h.label === "A")).toBe(false); + expect(records(rows)).toHaveLength(1); + }); + + it("groups by local day", () => { + const day = new Date(2026, 7, 9, 12).getTime(); + const nextDay = new Date(2026, 7, 10, 12).getTime(); + const rows = buildGroupedRows( + [ + record({ id: 1, timestampEpoch: day }), + record({ id: 2, timestampEpoch: day }), + record({ id: 3, timestampEpoch: nextDay }), + ], + ["day"], + new Set() + ); + expect(groups(rows).map((h) => h.count)).toEqual([2, 1]); + }); + + it("labels missing values instead of showing an empty header", () => { + const rows = buildGroupedRows([record({ provider: "" })], ["provider"], new Set()); + expect(groups(rows)[0].label).toBe("(no provider)"); + }); + + it("handles an empty record set", () => { + expect(buildGroupedRows([], ["level"], new Set())).toEqual([]); + }); +}); + +describe("allGroupKeys", () => { + it("returns every key including nested ones", () => { + const keys = allGroupKeys( + [ + record({ id: 1, level: "Error", provider: "A" }), + record({ id: 2, level: "Warning", provider: "B" }), + ], + ["level", "provider"] as EvtxGroupField[] + ); + expect(keys.size).toBe(4); + }); + + it("is empty when nothing is grouped", () => { + expect(allGroupKeys([record({})], []).size).toBe(0); + }); +}); + +describe("group key encoding", () => { + it("keeps distinct ancestries distinct when a value contains the delimiters", () => { + // Unencoded, a provider literally named "x/level=Error" produces the same key as the + // Error subgroup of provider "x", and collapsing one would collapse the other. + const rows = buildGroupedRows( + [ + record({ id: 1, provider: "x/level=Error", level: "Information" }), + record({ id: 2, provider: "x", level: "Error" }), + ], + ["provider", "level"], + new Set() + ); + + const keys = rows.flatMap((r) => (r.kind === "group" ? [r.key] : [])); + expect(new Set(keys).size).toBe(keys.length); + }); + + it("collapsing one group does not collapse an unrelated one", () => { + const all = buildGroupedRows( + [ + record({ id: 1, provider: "x/level=Error", level: "Information" }), + record({ id: 2, provider: "x", level: "Error" }), + ], + ["provider", "level"], + new Set() + ); + const target = all.find((r) => r.kind === "group" && r.label === "Error"); + expect(target?.kind).toBe("group"); + const targetKey = target && target.kind === "group" ? target.key : ""; + + const collapsed = buildGroupedRows( + [ + record({ id: 1, provider: "x/level=Error", level: "Information" }), + record({ id: 2, provider: "x", level: "Error" }), + ], + ["provider", "level"], + new Set([targetKey]) + ); + + // Exactly one record disappears; the other ancestry is untouched. + const visible = collapsed.filter((r) => r.kind === "record"); + expect(visible).toHaveLength(1); + expect(visible[0].record.id).toBe(1); + }); +}); + +describe("day grouping honours the time zone", () => { + it("buckets by UTC date when asked for UTC", () => { + // Every other test here relies on the default, so a regression that ignored the argument and + // always bucketed by local date would pass the whole file. This instant sits on either side of + // midnight depending on the zone. + const lateUtc = Date.UTC(2026, 1, 10, 23, 30, 0); + const rows = buildGroupedRows( + [record({ id: 1, timestampEpoch: lateUtc })], + ["day"], + new Set(), + "utc" + ); + const header = rows.find((r) => r.kind === "group"); + expect(header && header.kind === "group" ? header.label : "").toBe("2026-02-10"); + }); + + it("buckets by the machine's own date when asked for local", () => { + const lateUtc = Date.UTC(2026, 1, 10, 23, 30, 0); + const local = new Date(lateUtc); + const expected = + `${local.getFullYear()}-${String(local.getMonth() + 1).padStart(2, "0")}-` + + `${String(local.getDate()).padStart(2, "0")}`; + const rows = buildGroupedRows( + [record({ id: 1, timestampEpoch: lateUtc })], + ["day"], + new Set(), + "local" + ); + const header = rows.find((r) => r.kind === "group"); + expect(header && header.kind === "group" ? header.label : "").toBe(expected); + }); +}); diff --git a/src/workspaces/event-log/evtx-saved-filters.test.ts b/src/workspaces/event-log/evtx-saved-filters.test.ts new file mode 100644 index 000000000..1c35850cb --- /dev/null +++ b/src/workspaces/event-log/evtx-saved-filters.test.ts @@ -0,0 +1,189 @@ +import { describe, expect, it } from "vitest"; +import { + buildFilterExport, + mergeFilters, + orderFilters, + parseFilterExport, + SAVED_FILTER_SCHEMA, + ALL_LEVELS, + sanitizeCriteria, + sanitizeSavedFilter, + type EvtxSavedFilter, +} from "./evtx-saved-filters"; + +function filter(partial: Partial): EvtxSavedFilter { + return { + id: "id", + name: "name", + favorite: false, + tags: [], + lastUsed: null, + criteria: sanitizeCriteria({}), + ...partial, + }; +} + +describe("sanitizeCriteria", () => { + it("keeps recognised values", () => { + const criteria = sanitizeCriteria({ + levels: ["Error", "Warning"], + eventIds: "4624,4625", + search: "logon", + timeWindow: "7d", + groupBy: ["level", "provider"], + }); + expect(criteria.levels).toEqual(["Error", "Warning"]); + expect(criteria.timeWindow).toBe("7d"); + expect(criteria.groupBy).toEqual(["level", "provider"]); + }); + + it("drops unrecognised levels and group fields rather than trusting the file", () => { + // Import files are hand-edited and shared; an unknown value must not widen a filter. + const criteria = sanitizeCriteria({ + levels: ["Error", "Bogus"], + groupBy: ["level", "nonsense"], + }); + expect(criteria.levels).toEqual(["Error"]); + expect(criteria.groupBy).toEqual(["level"]); + }); + + it("falls back to every level when none survive, rather than matching nothing", () => { + const criteria = sanitizeCriteria({ levels: ["Bogus"] }); + // Compared against the constant rather than its current size, so adding a level does not + // report a length mismatch that says nothing about the cause. + expect([...criteria.levels].sort()).toEqual([...ALL_LEVELS].sort()); + }); + + it("falls back to a known time window", () => { + expect(sanitizeCriteria({ timeWindow: "forever" }).timeWindow).toBe("24h"); + }); + + it("tolerates a completely wrong shape", () => { + expect(sanitizeCriteria(null).eventIds).toBe(""); + expect(sanitizeCriteria(42).search).toBe(""); + expect(sanitizeCriteria(["a"]).levels).toHaveLength(5); + }); +}); + +describe("sanitizeSavedFilter", () => { + it("requires a name", () => { + expect(sanitizeSavedFilter({ name: " " }, "fallback")).toBeNull(); + expect(sanitizeSavedFilter({}, "fallback")).toBeNull(); + }); + + it("supplies a fallback id and dedupes tags", () => { + const saved = sanitizeSavedFilter( + { name: "Logons", tags: ["auth", "auth", " auth ", ""] }, + "fallback" + ); + expect(saved?.id).toBe("fallback"); + expect(saved?.tags).toEqual(["auth"]); + }); + + it("treats a non-boolean favorite as not favorite", () => { + expect(sanitizeSavedFilter({ name: "x", favorite: "yes" }, "f")?.favorite).toBe(false); + }); +}); + +describe("parseFilterExport", () => { + it("round-trips through buildFilterExport", () => { + const original = [filter({ id: "a", name: "Errors", favorite: true })]; + const { filters, skipped } = parseFilterExport(buildFilterExport(original)); + expect(skipped).toBe(0); + expect(filters[0].name).toBe("Errors"); + expect(filters[0].favorite).toBe(true); + }); + + it("skips individually invalid entries instead of failing the whole import", () => { + // One bad entry in a shared file must not cost the operator the rest of it. + const text = JSON.stringify({ + schema: 1, + filters: [{ name: "Good" }, { noName: true }, { name: "Also good" }], + }); + const { filters, skipped } = parseFilterExport(text); + expect(filters.map((f) => f.name)).toEqual(["Good", "Also good"]); + expect(skipped).toBe(1); + }); + + it("returns nothing for malformed json rather than throwing", () => { + expect(parseFilterExport("{not json").filters).toEqual([]); + expect(parseFilterExport("[]").filters).toEqual([]); + }); +}); + +describe("mergeFilters", () => { + it("replaces a same-named filter and keeps the existing id", () => { + // Ids are per machine, so matching on them would duplicate a filter shared twice. + const existing = [filter({ id: "local", name: "Errors" })]; + const imported = [filter({ id: "remote", name: "errors", favorite: true })]; + const merged = mergeFilters(existing, imported); + expect(merged).toHaveLength(1); + expect(merged[0].id).toBe("local"); + expect(merged[0].favorite).toBe(true); + }); + + it("appends filters that do not collide", () => { + const merged = mergeFilters( + [filter({ id: "a", name: "A" })], + [filter({ id: "b", name: "B" })] + ); + expect(merged.map((f) => f.name)).toEqual(["A", "B"]); + }); +}); + +describe("orderFilters", () => { + it("puts favorites first, then most recent, then alphabetical", () => { + const ordered = orderFilters([ + filter({ name: "Zulu" }), + filter({ name: "Alpha" }), + filter({ name: "Recent", lastUsed: 100 }), + filter({ name: "Starred", favorite: true }), + ]); + expect(ordered.map((f) => f.name)).toEqual(["Starred", "Recent", "Alpha", "Zulu"]); + }); + + it("does not mutate its input", () => { + const input = [filter({ name: "B" }), filter({ name: "A" })]; + orderFilters(input); + expect(input.map((f) => f.name)).toEqual(["B", "A"]); + }); +}); + +describe("hostile stored values", () => { + it("rejects a non-finite lastUsed", () => { + // JSON admits 1e309, which parses to Infinity and would reach the ordering comparator as a + // non-finite operand. + // Literal JSON text, not JSON.stringify: stringify writes 1e309 as null, so the assertion + // held whether or not the sanitizer rejected Infinity. JSON.parse produces Infinity. + const parsed = parseFilterExport( + '{"schema":1,"filters":[{"id":"a","name":"A","lastUsed":1e309}]}' + ); + expect(parsed.filters).toHaveLength(1); + expect(parsed.filters[0].lastUsed).toBeNull(); + }); + + it("keeps a finite lastUsed", () => { + const parsed = parseFilterExport( + JSON.stringify({ schema: 1, filters: [{ id: "a", name: "A", lastUsed: 1700000000000 }] }) + ); + expect(parsed.filters[0].lastUsed).toBe(1700000000000); + }); + + it("refuses an export written by a newer schema", () => { + // Sanitizing it into whatever this build understands would import silently and quietly change + // the operator's criteria rather than saying the file is from a later version. + const parsed = parseFilterExport( + JSON.stringify({ schema: 99, filters: [{ id: "a", name: "A" }] }) + ); + expect(parsed.filters).toHaveLength(0); + expect(parsed.unsupportedSchema).toBe(true); + }); + + it("still accepts the current schema", () => { + const parsed = parseFilterExport( + JSON.stringify({ schema: SAVED_FILTER_SCHEMA, filters: [{ id: "a", name: "A" }] }) + ); + expect(parsed.filters).toHaveLength(1); + expect(parsed.unsupportedSchema).toBeUndefined(); + }); +}); diff --git a/src/workspaces/event-log/evtx-saved-filters.ts b/src/workspaces/event-log/evtx-saved-filters.ts new file mode 100644 index 000000000..1578fd835 --- /dev/null +++ b/src/workspaces/event-log/evtx-saved-filters.ts @@ -0,0 +1,197 @@ +/** + * Saved event-log filters. + * + * EventLogExpert and Event Log Explorer both keep a filter library; we had none, so every + * investigation started by retyping the same criteria. Kept separate from the store so the + * serialization rules are testable without a Zustand or Tauri runtime. + */ +import type { EvtxLevel, EvtxTimeWindow } from "./types"; +import type { EvtxGroupField } from "./evtx-filter"; + +/** Everything a saved filter restores. */ +export interface EvtxFilterCriteria { + levels: EvtxLevel[]; + eventIds: string; + search: string; + timeWindow: EvtxTimeWindow; + groupBy: EvtxGroupField[]; +} + +export interface EvtxSavedFilter { + id: string; + name: string; + favorite: boolean; + tags: string[]; + criteria: EvtxFilterCriteria; + /** Epoch millis of the last apply, for a recents list. Null when never applied. */ + lastUsed: number | null; +} + +/** Every level, and the fallback when a stored filter names none this build recognizes. */ +export const ALL_LEVELS: EvtxLevel[] = [ + "Critical", + "Error", + "Warning", + "Information", + "Verbose", +]; +const TIME_WINDOWS: EvtxTimeWindow[] = ["1h", "24h", "7d", "30d", "all"]; +const GROUP_FIELDS: EvtxGroupField[] = [ + "level", + "provider", + "channel", + "eventId", + "day", +]; + +/** The current schema version, so an older export can be recognised rather than misread. */ +export const SAVED_FILTER_SCHEMA = 1; + +export interface EvtxFilterExport { + schema: number; + filters: EvtxSavedFilter[]; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** + * Coerces untrusted input into criteria, dropping anything unrecognised. + * + * Import files are edited by hand and shared between machines, so every field is validated against + * the values this build knows. An unknown level silently widening a filter would be worse than + * dropping it: the operator would believe they were filtering when they were not. + */ +export function sanitizeCriteria(input: unknown): EvtxFilterCriteria { + const raw = isRecord(input) ? input : {}; + + const levels = Array.isArray(raw.levels) + ? (raw.levels.filter((level): level is EvtxLevel => + ALL_LEVELS.includes(level as EvtxLevel) + ) as EvtxLevel[]) + : []; + + const groupBy = Array.isArray(raw.groupBy) + ? (raw.groupBy.filter((field): field is EvtxGroupField => + GROUP_FIELDS.includes(field as EvtxGroupField) + ) as EvtxGroupField[]) + : []; + + return { + // An empty level list would match nothing, which no operator means to save, so it falls back + // to every level rather than producing a filter that silently hides everything. + levels: levels.length > 0 ? levels : [...ALL_LEVELS], + eventIds: typeof raw.eventIds === "string" ? raw.eventIds : "", + search: typeof raw.search === "string" ? raw.search : "", + timeWindow: TIME_WINDOWS.includes(raw.timeWindow as EvtxTimeWindow) + ? (raw.timeWindow as EvtxTimeWindow) + : "24h", + groupBy, + }; +} + +/** Coerces untrusted input into a saved filter, or null when it carries no usable name. */ +export function sanitizeSavedFilter(input: unknown, fallbackId: string): EvtxSavedFilter | null { + if (!isRecord(input)) return null; + const name = typeof input.name === "string" ? input.name.trim() : ""; + if (!name) return null; + + return { + id: typeof input.id === "string" && input.id ? input.id : fallbackId, + name, + favorite: input.favorite === true, + tags: Array.isArray(input.tags) + ? Array.from( + new Set( + input.tags + .filter((tag): tag is string => typeof tag === "string") + .map((tag) => tag.trim()) + .filter(Boolean) + ) + ) + : [], + criteria: sanitizeCriteria(input.criteria), + // Finite only. JSON admits 1e309, which parses to Infinity and would reach the ordering + // comparator as a non-finite operand. + lastUsed: typeof input.lastUsed === "number" && Number.isFinite(input.lastUsed) + ? input.lastUsed + : null, + }; +} + +/** + * Parses an exported filter file. + * + * Individually invalid filters are skipped rather than failing the whole import, so one bad entry + * in a shared file does not cost the operator the rest of it. + */ +export function parseFilterExport(text: string): { + filters: EvtxSavedFilter[]; + skipped: number; + /** True when the file was written by a build using a schema this one does not know. */ + unsupportedSchema?: boolean; +} { + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + return { filters: [], skipped: 0 }; + } + + // The schema is written on export and must be checked on import. A newer file would otherwise be + // sanitized into whatever this build understands and imported silently, quietly changing the + // operator's criteria rather than telling them the file is from a later version. + if (isRecord(parsed) && parsed.schema !== undefined && parsed.schema !== SAVED_FILTER_SCHEMA) { + return { filters: [], skipped: 0, unsupportedSchema: true }; + } + + const list = isRecord(parsed) && Array.isArray(parsed.filters) ? parsed.filters : []; + const filters: EvtxSavedFilter[] = []; + let skipped = 0; + + list.forEach((entry, index) => { + const filter = sanitizeSavedFilter(entry, `imported-${index}`); + if (filter) filters.push(filter); + else skipped += 1; + }); + + return { filters, skipped }; +} + +/** Serializes filters for export. */ +export function buildFilterExport(filters: EvtxSavedFilter[]): string { + const payload: EvtxFilterExport = { schema: SAVED_FILTER_SCHEMA, filters }; + return JSON.stringify(payload, null, 2); +} + +/** + * Merges imported filters into an existing library. + * + * Matching is by name rather than id, because ids are generated per machine and the same filter + * shared twice would otherwise accumulate duplicates. An import replaces a same-named filter, since + * the operator chose to import it. + */ +export function mergeFilters( + existing: EvtxSavedFilter[], + imported: EvtxSavedFilter[] +): EvtxSavedFilter[] { + const merged = [...existing]; + for (const filter of imported) { + const index = merged.findIndex( + (candidate) => candidate.name.toLowerCase() === filter.name.toLowerCase() + ); + if (index >= 0) merged[index] = { ...filter, id: merged[index].id }; + else merged.push(filter); + } + return merged; +} + +/** Favorites first, then most recently used, then by name. */ +export function orderFilters(filters: EvtxSavedFilter[]): EvtxSavedFilter[] { + return [...filters].sort((a, b) => { + if (a.favorite !== b.favorite) return a.favorite ? -1 : 1; + if (a.lastUsed !== b.lastUsed) return (b.lastUsed ?? 0) - (a.lastUsed ?? 0); + return a.name.localeCompare(b.name); + }); +} diff --git a/src/workspaces/event-log/evtx-store-coverage.test.ts b/src/workspaces/event-log/evtx-store-coverage.test.ts new file mode 100644 index 000000000..79ee065ad --- /dev/null +++ b/src/workspaces/event-log/evtx-store-coverage.test.ts @@ -0,0 +1,449 @@ +/** + * Coverage-gap handling through the store's real load paths. + * + * `evtx-coverage.test.ts` covers the merge rule in isolation. That is not enough: the rule can be + * right while a call site drops the gaps entirely, which is exactly what `queryChannels` did. These + * drive the store itself so a regression in a load path fails here. + * + * The Tauri bridge is mocked because the store imports it at module scope. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const invoke = vi.hoisted(() => vi.fn()); + +// The store subscribes to backend events at module scope. Capturing the handlers lets a test +// deliver a batch exactly as the backend would, including delivering none. +const listeners = vi.hoisted(() => new Map void>()); +vi.mock("@tauri-apps/api/core", () => ({ invoke })); +vi.mock("@tauri-apps/api/event", () => ({ + listen: vi.fn((name: string, handler: (event: { payload: unknown }) => void) => { + listeners.set(name, handler); + return Promise.resolve(() => {}); + }), +})); + +const { useEvtxStore } = await import("./evtx-store"); + +/** Delivers a batch the way the backend emits one. */ +function emitBatch(channel: string, sequence: number, records: unknown[]) { + listeners.get("evtx-record-batch")?.({ payload: { channel, sequence, records } }); +} + +function result(channel: string, gaps: string[]) { + return { + records: [], + channels: [{ name: channel, eventCount: 0, sourceType: "live" }], + totalRecords: 0, + parseErrors: gaps.length, + errorMessages: gaps, + }; +} + +describe("coverage gaps through the store", () => { + beforeEach(() => { + invoke.mockReset(); + useEvtxStore.setState({ + records: [], + channels: [], + coverageGaps: [], + loadedChannels: new Set(), + selectedChannels: new Set(), + }); + }); + + it("keeps the gaps a channel query reported", async () => { + // queryChannels previously discarded these, so a partly unreadable channel looked complete. + invoke.mockResolvedValueOnce(result("Application", ["Application: 3 records unreadable"])); + + await useEvtxStore.getState().queryChannels(["Application"]); + + expect(useEvtxStore.getState().coverageGaps).toEqual([ + "Application: 3 records unreadable", + ]); + }); + + it("accumulates gaps as channels load one at a time", async () => { + invoke + .mockResolvedValueOnce(result("Application", ["Application: 3 records unreadable"])) + .mockResolvedValueOnce(result("System", ["System: stopped at 100000 events"])); + + await useEvtxStore.getState().queryChannels(["Application"]); + await useEvtxStore.getState().queryChannels(["System"]); + + expect(useEvtxStore.getState().coverageGaps).toHaveLength(2); + }); + + it("does not repeat a gap when the same channel is queried again", async () => { + invoke + .mockResolvedValueOnce(result("Application", ["Application: 3 records unreadable"])) + .mockResolvedValueOnce(result("Application", ["Application: 3 records unreadable"])); + + await useEvtxStore.getState().queryChannels(["Application"]); + await useEvtxStore.getState().queryChannels(["Application"]); + + expect(useEvtxStore.getState().coverageGaps).toHaveLength(1); + }); + + it("reports nothing when a channel loads cleanly", async () => { + invoke.mockResolvedValueOnce(result("Application", [])); + + await useEvtxStore.getState().queryChannels(["Application"]); + + expect(useEvtxStore.getState().coverageGaps).toEqual([]); + }); + + it("a refresh drops gaps from the records it replaced", async () => { + // The refresh clears the records, so gaps describing them must go too, or the banner reports a + // gap in a set that is no longer on screen. + invoke.mockResolvedValueOnce(result("Application", ["Application: stale gap"])); + await useEvtxStore.getState().queryChannels(["Application"]); + expect(useEvtxStore.getState().coverageGaps).toHaveLength(1); + + invoke.mockResolvedValueOnce(result("Application", ["Application: fresh gap"])); + await useEvtxStore.getState().refreshLoadedChannels(); + + expect(useEvtxStore.getState().coverageGaps).toEqual(["Application: fresh gap"]); + }); + + it("records a channel whose refresh failed instead of showing the cleared view as complete", async () => { + // The refresh clears coverageGaps with the records it replaced. A channel whose refresh then + // fails contributes zero records to the replaced view, so the failure must be recorded or the + // view reports full coverage while missing a whole channel. + invoke.mockResolvedValueOnce(result("Application", [])); + await useEvtxStore.getState().queryChannels(["Application"]); + + invoke.mockRejectedValueOnce(new Error("access denied")); + await useEvtxStore.getState().refreshLoadedChannels(); + + const state = useEvtxStore.getState(); + expect( + state.coverageGaps.some((g) => g.includes("Application") && g.includes("access denied")) + ).toBe(true); + expect(state.loadError).toContain("Application"); + expect(state.loadError).toContain("access denied"); + expect(state.isLoading).toBe(false); + }); +}); + +describe("a multi-channel query is delivered one channel at a time", () => { + beforeEach(() => { + invoke.mockReset(); + useEvtxStore.setState({ + records: [], + channels: [], + coverageGaps: [], + loadedChannels: new Set(), + selectedChannels: new Set(), + }); + }); + + function withRecords(channel: string, count: number) { + return { + records: Array.from({ length: count }, (_, i) => ({ + id: i, + eventRecordId: i, + timestamp: "2026-08-11T12:00:00.000Z", + timestampEpoch: 1_000 + i, + provider: "P", + channel, + eventId: 1, + level: "information", + computer: "C", + message: "m", + eventData: [], + rawXml: "", + sourceLabel: "Live", + mapped: null, + })), + channels: [{ name: channel, eventCount: count, sourceType: "live" }], + totalRecords: count, + parseErrors: 0, + errorMessages: [], + }; + } + + it("asks for each channel separately rather than all of them at once", async () => { + // The backend collects a whole request into one vector before replying, so a single request + // naming every channel holds every event of every channel in memory before anything is shown. + invoke + .mockResolvedValueOnce(withRecords("Application", 1)) + .mockResolvedValueOnce(withRecords("System", 1)); + + await useEvtxStore.getState().queryChannels(["Application", "System"]); + + expect(invoke).toHaveBeenCalledTimes(2); + const requested = invoke.mock.calls.map( + (call) => (call[1] as { channels: string[] }).channels + ); + expect(requested).toEqual([["Application"], ["System"]]); + }); + + it("keeps the channels that succeeded when one of them fails", async () => { + // A single request fails as a whole. One unreadable channel used to discard the results of + // every channel queried alongside it and leave the view empty. + invoke + .mockRejectedValueOnce(new Error("access denied")) + .mockResolvedValueOnce(withRecords("System", 3)); + + await useEvtxStore.getState().queryChannels(["Security", "System"]); + + const state = useEvtxStore.getState(); + expect(state.records).toHaveLength(3); + expect(state.loadedChannels.has("System")).toBe(true); + expect(state.loadedChannels.has("Security")).toBe(false); + }); + + it("records the unreadable channel as a gap, not only as an error string", async () => { + // loadError is replaced by the next load. A gap describes the view currently on screen, and + // events that never arrived look exactly like evidence that nothing happened. + invoke + .mockRejectedValueOnce(new Error("access denied")) + .mockResolvedValueOnce(withRecords("System", 1)); + + await useEvtxStore.getState().queryChannels(["Security", "System"]); + + const gaps = useEvtxStore.getState().coverageGaps; + expect(gaps.some((g) => g.includes("Security") && g.includes("access denied"))).toBe(true); + }); + + it("still reports the failure through loadError", async () => { + invoke.mockRejectedValueOnce(new Error("access denied")); + + await useEvtxStore.getState().queryChannels(["Security"]); + + expect(useEvtxStore.getState().loadError).toContain("Security"); + expect(useEvtxStore.getState().isLoading).toBe(false); + }); +}); + +describe("the time window reaches the service", () => { + beforeEach(() => { + invoke.mockReset(); + useEvtxStore.setState({ + records: [], + channels: [], + coverageGaps: [], + loadedChannels: new Set(), + selectedChannels: new Set(), + timeWindow: "1h", + }); + }); + + const filterOf = (call: number) => + (invoke.mock.calls[call]?.[1] as { filter?: { time?: { milliseconds: number } } })?.filter; + + it("sends the selected window when a channel is queried", () => { + invoke.mockResolvedValueOnce(result("Application", [])); + return useEvtxStore + .getState() + .queryChannels(["Application"]) + .then(() => { + expect(filterOf(0)?.time?.milliseconds).toBe(60 * 60 * 1000); + }); + }); + + it("sends it on refresh too", async () => { + // The window is a server-side predicate and a refresh is the only thing that applies it. + // Omitting it made the control a no-op: selecting 1h triggered a refresh that then fetched the + // channel unbounded, so the view filled with events outside the window still shown as selected. + invoke.mockResolvedValueOnce(result("Application", [])); + await useEvtxStore.getState().queryChannels(["Application"]); + + invoke.mockResolvedValueOnce(result("Application", [])); + await useEvtxStore.getState().refreshLoadedChannels(); + + expect(filterOf(1)?.time?.milliseconds).toBe(60 * 60 * 1000); + }); + + it("sends no time predicate when the window is all time", async () => { + useEvtxStore.setState({ timeWindow: "all" }); + invoke.mockResolvedValueOnce(result("Application", [])); + await useEvtxStore.getState().queryChannels(["Application"]); + + expect(filterOf(0)?.time).toBeUndefined(); + }); + + it("drops a non-string gap the reader sent rather than rendering it", () => { + // The guard normalizes, but only if callers use what it returns. Ignoring the return value + // left the raw reply in place and stored 42 in coverageGaps. + invoke.mockResolvedValueOnce({ + records: [], + channels: [], + totalRecords: 0, + parseErrors: 1, + errorMessages: ["real gap", 42, null], + }); + + return useEvtxStore + .getState() + .queryChannels(["Application"]) + .then(() => { + expect(useEvtxStore.getState().coverageGaps).toEqual(["real gap"]); + }); + }); +}); + +describe("records that arrive in batches while the query runs", () => { + beforeEach(() => { + invoke.mockReset(); + useEvtxStore.setState({ + records: [], + channels: [], + coverageGaps: [], + loadedChannels: new Set(), + selectedChannels: new Set(), + }); + }); + + function record(channel: string, epoch: number) { + return { + id: 0, + eventRecordId: epoch, + timestamp: "2026-08-12T12:00:00.000Z", + timestampEpoch: epoch, + provider: "P", + channel, + eventId: 1, + level: "information", + computer: "C", + message: "m", + eventData: [], + rawXml: "", + sourceLabel: "Live", + mapped: null, + }; + } + + /** A reply that streamed everything: it carries the count but none of the records. */ + function streamedReply(channel: string, totalRecords: number) { + return { + records: [], + channels: [{ name: channel, eventCount: totalRecords, sourceType: "live" }], + totalRecords, + parseErrors: 0, + errorMessages: [], + }; + } + + it("assembles the view from batches the reply did not carry", async () => { + invoke.mockImplementationOnce(async () => { + emitBatch("System", 0, [record("System", 1), record("System", 2)]); + emitBatch("System", 1, [record("System", 3)]); + return streamedReply("System", 3); + }); + + await useEvtxStore.getState().queryChannels(["System"]); + + const state = useEvtxStore.getState(); + expect(state.records).toHaveLength(3); + expect(state.coverageGaps).toEqual([]); + }); + + it("reports a batch that never arrived instead of showing a short list as complete", async () => { + // Sequence 1 is skipped. Its events are simply absent, and an absent event is indistinguishable + // from an event that never happened unless the gap is stated. + invoke.mockImplementationOnce(async () => { + emitBatch("System", 0, [record("System", 1)]); + emitBatch("System", 2, [record("System", 3)]); + return streamedReply("System", 3); + }); + + await useEvtxStore.getState().queryChannels(["System"]); + + const gaps = useEvtxStore.getState().coverageGaps; + expect(gaps.some((g) => g.includes("System") && g.includes("batches"))).toBe(true); + }); + + it("reports a shortfall against the count the reader sent", async () => { + // Every batch arrived in order, but fewer events than the reader says it sent. + invoke.mockImplementationOnce(async () => { + emitBatch("System", 0, [record("System", 1)]); + return streamedReply("System", 9); + }); + + await useEvtxStore.getState().queryChannels(["System"]); + + const gaps = useEvtxStore.getState().coverageGaps; + expect(gaps.some((g) => g.includes("8 of 9"))).toBe(true); + }); + + it("does not invent a shortfall when the reader gave no count", async () => { + // An absent count means completeness cannot be checked. Treating it as zero would report every + // arriving record as unexpected; treating it as a shortfall would cry wolf on every load. + invoke.mockImplementationOnce(async () => { + emitBatch("System", 0, [record("System", 1)]); + return { ...streamedReply("System", 0), totalRecords: "unknown" }; + }); + + await useEvtxStore.getState().queryChannels(["System"]); + + expect(useEvtxStore.getState().records).toHaveLength(1); + expect(useEvtxStore.getState().coverageGaps).toEqual([]); + }); + + it("still works when the reader returned the records in the reply instead", async () => { + // Collecting callers exist, and a backend that did not stream must not look like a total loss. + invoke.mockResolvedValueOnce({ + records: [record("System", 1), record("System", 2)], + channels: [{ name: "System", eventCount: 2, sourceType: "live" }], + totalRecords: 2, + parseErrors: 0, + errorMessages: [], + }); + + await useEvtxStore.getState().queryChannels(["System"]); + + expect(useEvtxStore.getState().records).toHaveLength(2); + expect(useEvtxStore.getState().coverageGaps).toEqual([]); + }); + + it("does not count a previous attempt's batches towards a retry", async () => { + invoke.mockImplementationOnce(async () => { + emitBatch("System", 0, [record("System", 1)]); + throw new Error("interrupted"); + }); + await useEvtxStore.getState().queryChannels(["System"]); + + useEvtxStore.setState({ records: [], coverageGaps: [], loadedChannels: new Set() }); + invoke.mockImplementationOnce(async () => { + emitBatch("System", 0, [record("System", 5)]); + return streamedReply("System", 1); + }); + await useEvtxStore.getState().queryChannels(["System"]); + + const state = useEvtxStore.getState(); + expect(state.records).toHaveLength(1); + expect(state.records[0].eventRecordId).toBe(5); + }); + + it("finds the highest batch number without spreading every one", async () => { + // The highest sequence is found by reduction. Spreading the set into Math.max(...) arguments + // throws RangeError once a channel produces more batches than the engine accepts as arguments, + // which a reduced fetch batch makes reachable for a channel the size of Security. + const batches = 200_000; + invoke.mockImplementationOnce(async () => { + for (let sequence = 0; sequence < batches; sequence++) { + emitBatch("Huge", sequence, []); + } + return streamedReply("Huge", 0); + }); + + await useEvtxStore.getState().queryChannels(["Huge"]); + + expect(useEvtxStore.getState().coverageGaps).toEqual([]); + }); + + it("does not leave the workspace stuck loading when a reply is unreadable", async () => { + // assertParseResultShape throws by design on a reply this build cannot read. The processing + // loop was unguarded, so the throw rejected queryChannels before isLoading was cleared and the + // operator saw an endless spinner with no error. + invoke.mockResolvedValueOnce({ records: null, channels: [] }); + + await useEvtxStore.getState().queryChannels(["Application"]); + + const state = useEvtxStore.getState(); + expect(state.isLoading).toBe(false); + expect(state.loadError).toContain("Application"); + expect(state.coverageGaps.some((g) => g.includes("Application"))).toBe(true); + }); +}); diff --git a/src/workspaces/event-log/evtx-store.ts b/src/workspaces/event-log/evtx-store.ts index 1c9a60d42..304a1e13e 100644 --- a/src/workspaces/event-log/evtx-store.ts +++ b/src/workspaces/event-log/evtx-store.ts @@ -1,4 +1,6 @@ import { create } from "zustand"; +import { assertParseResultShape, mergeCoverageGaps } from "./evtx-coverage"; +import type { EvtxTimeZoneMode } from "./evtx-time"; import { invoke } from "@tauri-apps/api/core"; import { listen } from "@tauri-apps/api/event"; import type { @@ -6,7 +8,34 @@ import type { EvtxChannelInfo, EvtxLevel, EvtxParseResult, + EvtxTimeWindow, + EventQueryFilterSubset, } from "./types"; +import { EVTX_TIME_WINDOW_MS } from "./types"; + +// Re-exported so callers have one import site; the implementations live in a Tauri-free module. +export { parseEventIdFilter, selectVisibleRecords } from "./evtx-filter"; +import type { EvtxGroupField } from "./evtx-filter"; +import { + defaultColumnConfig, + moveColumn, + sanitizeColumnConfig, + toggleColumn, + type EvtxColumnConfig, + type EvtxColumnId, +} from "./evtx-columns"; + +/** + * Builds the filter handed to the backend, which compiles it to XPath. + * + * Only the time window is pushed down today. Level and provider stay client-side because the + * existing controls filter records already in memory; moving them server-side changes what a + * reload fetches, which is a behavioural change worth making deliberately rather than implicitly. + */ +function buildServerFilter(timeWindow: EvtxTimeWindow): EventQueryFilterSubset { + if (timeWindow === "all") return {}; + return { time: { kind: "last", milliseconds: EVTX_TIME_WINDOW_MS[timeWindow] } }; +} export type EvtxSourceMode = "files" | "live" | null; export type EvtxSortField = "time" | "eventId" | "level" | "provider" | "channel"; @@ -24,11 +53,30 @@ interface EvtxState { loadStartTime: number | null; loadElapsedMs: number | null; loadError: string | null; + /** + * What is missing from the loaded set, and why. + * + * Separate from loadError because these are not failures: the events that did load are real and + * usable. They are gaps, and a gap that only reaches the console reads to an operator as a + * complete picture, which is how absent events get mistaken for evidence that nothing happened. + */ + coverageGaps: string[]; selectedChannels: Set; loadedChannels: Set; filterLevels: Set; filterEventIds: string; filterSearch: string; + timeWindow: EvtxTimeWindow; + /** + * Which clock event times are shown in. + * + * Defaults to local, which is the clock an admin comparing against a user's report is thinking + * in. UTC is one click away because that is the clock every other log in an escalation is in. + */ + timeZoneMode: EvtxTimeZoneMode; + columnConfig: EvtxColumnConfig; + groupBy: EvtxGroupField[]; + collapsedGroups: Set; sortField: EvtxSortField; sortDirection: EvtxSortDirection; selectedRecordId: number | null; @@ -38,6 +86,7 @@ interface EvtxState { queryChannels: (channels: string[], maxEvents?: number) => Promise; loadSelectedChannels: () => Promise; refreshLoadedChannels: () => Promise; + setTimeZoneMode: (mode: EvtxTimeZoneMode) => void; setSelectedChannels: (channels: Set) => void; toggleChannel: (channel: string) => void; selectAllChannels: () => void; @@ -46,6 +95,12 @@ interface EvtxState { toggleFilterLevel: (level: EvtxLevel) => void; setFilterEventIds: (eventIds: string) => void; setFilterSearch: (search: string) => void; + setTimeWindow: (window: EvtxTimeWindow) => void; + setGroupBy: (fields: EvtxGroupField[]) => void; + toggleColumnVisible: (id: EvtxColumnId) => void; + moveColumnBy: (id: EvtxColumnId, direction: -1 | 1) => void; + resetColumns: () => void; + toggleGroup: (key: string) => void; setSortField: (field: EvtxSortField) => void; setSortDirection: (direction: EvtxSortDirection) => void; setSelectedRecordId: (id: number | null) => void; @@ -63,6 +118,7 @@ function applyParseResult( sourceMode, isLoading: false, loadError: null, + coverageGaps: result.errorMessages, selectedChannels: channelNames, selectedRecordId: null, }; @@ -78,11 +134,17 @@ export const useEvtxStore = create()((set, get) => ({ loadStartTime: null, loadElapsedMs: null, loadError: null, + coverageGaps: [], selectedChannels: new Set(), loadedChannels: new Set(), filterLevels: new Set(ALL_LEVELS), filterEventIds: "", filterSearch: "", + timeZoneMode: "local" as EvtxTimeZoneMode, + timeWindow: "24h", + columnConfig: defaultColumnConfig(), + groupBy: [], + collapsedGroups: new Set(), sortField: "time", sortDirection: "asc", selectedRecordId: null, @@ -91,7 +153,8 @@ export const useEvtxStore = create()((set, get) => ({ set({ isLoading: true, loadError: null }); try { const result = await invoke("evtx_parse_files", { paths }); - set(applyParseResult(result, "files")); + const checked = assertParseResultShape(result); + set(applyParseResult({ ...result, errorMessages: checked.errorMessages }, "files")); } catch (error) { const message = error instanceof Error ? error.message : String(error); set({ isLoading: false, loadError: message }); @@ -121,6 +184,7 @@ export const useEvtxStore = create()((set, get) => ({ sourceMode: "live", isLoading: true, loadError: null, + coverageGaps: [], loadStartTime: startTime, loadElapsedMs: null, selectedChannels: selectedNames, @@ -130,8 +194,7 @@ export const useEvtxStore = create()((set, get) => ({ }); // Query all core channels in parallel (bypass queryChannels to avoid isLoading conflicts) - const mergeResult = (ch: string, result: EvtxParseResult) => { - console.log(`[evtx] ${ch}: got ${result.records.length} records, ${result.parseErrors} errors`, result.errorMessages); + const mergeResult = (ch: string, result: EvtxParseResult, gaps: string[]) => { const state = get(); const merged = [...state.records, ...result.records]; merged.sort((a, b) => a.timestampEpoch - b.timestampEpoch); @@ -150,6 +213,10 @@ export const useEvtxStore = create()((set, get) => ({ channels: newChannels, loadedChannels: newLoaded, loadElapsedMs: performance.now() - startTime, + // Channels load one at a time and each may report its own gaps, so they accumulate + // rather than replace. Deduplicated because re-querying a channel would otherwise + // repeat the same line. + coverageGaps: mergeCoverageGaps(state.coverageGaps, gaps), }); }; @@ -158,8 +225,10 @@ export const useEvtxStore = create()((set, get) => ({ const result = await invoke("evtx_query_channels", { channels: [ch], maxEvents: null, + filter: buildServerFilter(get().timeWindow), }); - mergeResult(ch, result); + const checked = assertParseResultShape(result); + mergeResult(ch, result, checked.errorMessages); } catch (e) { const msg = e instanceof Error ? e.message : String(e); console.warn(`[evtx] Failed to query ${ch}: ${msg}`); @@ -183,45 +252,120 @@ export const useEvtxStore = create()((set, get) => ({ }, queryChannels: async (channels, maxEvents) => { - set({ isLoading: true, loadError: null }); - try { - const result = await invoke("evtx_query_channels", { - channels, - maxEvents: maxEvents ?? null, - }); + set({ isLoading: true, loadError: null, selectedRecordId: null }); + + // One request per channel rather than one request for all of them. The backend collects a + // whole request's records into a single vector before replying, so asking for forty channels + // at once held every event of every channel in memory twice, once per channel and once in the + // combined vector, before anything reached the screen. + // + // It also isolates failure. A single request fails as a whole, so one unreadable channel threw + // away the results of every channel queried alongside it and left the view empty. + let loadError: string | null = null; + + // Anything left over from an earlier attempt at these channels is dropped, so a retry cannot + // count a previous run's batches towards this one. + resetStreamedRecords(channels); + + const results = await Promise.all( + channels.map(async (ch) => { + try { + const result = await invoke("evtx_query_channels", { + channels: [ch], + maxEvents: maxEvents ?? null, + filter: buildServerFilter(get().timeWindow), + }); + return { channel: ch, result, error: null as string | null }; + } catch (e) { + const message = e instanceof Error ? e.message : String(e); + console.warn(`[evtx] Failed to query ${ch}: ${message}`); + if (!loadError) loadError = `${ch}: ${message}`; + return { channel: ch, result: null, error: message }; + } + }) + ); - // Merge new records with existing ones (for incremental channel loading) - const state = get(); - const existingChannelNames = new Set(state.records.map((r) => r.channel)); - // Only add records from channels we don't already have - const newRecords = result.records.filter((r) => !existingChannelNames.has(r.channel)); - const merged = [...state.records, ...newRecords]; - merged.sort((a, b) => a.timestampEpoch - b.timestampEpoch); - // Reassign IDs - for (let i = 0; i < merged.length; i++) merged[i].id = i; - - // Update channel event counts - const countMap = new Map(result.channels.map((c) => [c.name, c.eventCount])); - const updatedChannels = state.channels.map((c) => ({ - ...c, - eventCount: countMap.get(c.name) ?? c.eventCount, - })); - - const newLoaded = new Set(state.loadedChannels); - for (const ch of channels) newLoaded.add(ch); + for (const { channel, result, error } of results) { + try { + if (!result) { + // A channel that could not be read is recorded as a gap, not merely as an error banner + // that the next successful load replaces. The events it would have contributed are absent + // from the view for as long as the view is on screen. + set((s) => ({ + coverageGaps: mergeCoverageGaps(s.coverageGaps, [ + `${channel}: not read (${error ?? "unknown error"})`, + ]), + })); + continue; + } - set({ - records: merged, - channels: updatedChannels, - loadedChannels: newLoaded, - isLoading: false, - loadError: null, - selectedRecordId: null, - }); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - set({ isLoading: false, loadError: message }); + const checked = assertParseResultShape(result); + + // The records travel as batches while the query runs; the reply carries only whatever the + // backend did not stream. Both are taken, so this works whether or not streaming happened. + const streamed = drainStreamedRecords(channel); + const arrived = [...streamed.records, ...result.records]; + + // The reply says how many records were sent. Silence is not agreement: if fewer arrived, or a + // batch number is missing from the run, those events are absent from the view and must be + // said so rather than left to look like events that never happened. + const gapsFound: string[] = []; + if (streamed.missingSequences.length > 0) { + gapsFound.push( + `${channel}: ${streamed.missingSequences.length} batches of events were not received` + ); + } + const expected = checked.totalRecords; + if (typeof expected === "number" && arrived.length < expected) { + gapsFound.push( + `${channel}: ${expected - arrived.length} of ${expected} events did not reach the view` + ); + } + + const state = get(); + const existingChannelNames = new Set(state.records.map((r) => r.channel)); + // Only add records from channels we don't already have + const newRecords = arrived.filter((r) => !existingChannelNames.has(r.channel)); + const merged = [...state.records, ...newRecords]; + merged.sort((a, b) => a.timestampEpoch - b.timestampEpoch); + // Reassign IDs + for (let i = 0; i < merged.length; i++) merged[i].id = i; + + // Update channel event counts + const countMap = new Map(result.channels.map((c) => [c.name, c.eventCount])); + const updatedChannels = state.channels.map((c) => ({ + ...c, + eventCount: countMap.get(c.name) ?? c.eventCount, + })); + + const newLoaded = new Set(state.loadedChannels); + newLoaded.add(channel); + + set({ + records: merged, + channels: updatedChannels, + loadedChannels: newLoaded, + // Accumulated, not dropped. This path loads channels incrementally, so discarding what the + // backend reported here would show a complete view of a partly unreadable set. + coverageGaps: mergeCoverageGaps(state.coverageGaps, [ + ...checked.errorMessages, + ...gapsFound, + ]), + }); + } catch (processingError) { + // assertParseResultShape throws by design on a reply this build cannot read, and a malformed + // reply is not a reason to leave the workspace stuck on a spinner with no message. + const message = + processingError instanceof Error ? processingError.message : String(processingError); + console.warn(`[evtx] Failed to process ${channel}: ${message}`); + if (!loadError) loadError = `${channel}: ${message}`; + set((s) => ({ + coverageGaps: mergeCoverageGaps(s.coverageGaps, [`${channel}: not read (${message})`]), + })); + } } + + set({ isLoading: false, loadError }); }, loadSelectedChannels: async () => { @@ -243,8 +387,12 @@ export const useEvtxStore = create()((set, get) => ({ loadedChannels: new Set(), selectedRecordId: null, isLoading: true, + loadError: null, loadStartTime: startTime, loadElapsedMs: null, + // Cleared with the records they describe. Keeping them would report gaps from a set that is + // no longer on screen, while the new results' own gaps went unreported. + coverageGaps: [], }); const promises = loaded.map(async (ch) => { @@ -252,7 +400,13 @@ export const useEvtxStore = create()((set, get) => ({ const result = await invoke("evtx_query_channels", { channels: [ch], maxEvents: null, + // The window is a server-side predicate and a refetch is the only thing that applies it. + // Omitting it here made the time-window control a no-op: selecting 1h triggered this + // refresh, which then fetched the channel unbounded and replaced the view with events + // outside the window the toolbar was still showing as selected. + filter: buildServerFilter(get().timeWindow), }); + const checked = assertParseResultShape(result); const s = get(); const merged = [...s.records, ...result.records]; @@ -272,9 +426,17 @@ export const useEvtxStore = create()((set, get) => ({ channels: newChannels, loadedChannels: newLoaded, loadElapsedMs: performance.now() - startTime, + coverageGaps: mergeCoverageGaps(s.coverageGaps, checked.errorMessages), }); } catch (e) { - console.warn(`[evtx] Refresh failed for ${ch}:`, e); + const message = e instanceof Error ? e.message : String(e); + console.warn(`[evtx] Refresh failed for ${ch}: ${message}`); + // Recorded, not only logged. The refresh cleared the previous gaps, so a silent failure + // here presents a view that is missing a whole channel as complete. + set((s) => ({ + coverageGaps: mergeCoverageGaps(s.coverageGaps, [`${ch}: not read (${message})`]), + loadError: s.loadError ?? `${ch}: ${message}`, + })); } }); @@ -287,6 +449,8 @@ export const useEvtxStore = create()((set, get) => ({ }); }, + setTimeZoneMode: (mode) => set({ timeZoneMode: mode }), + setSelectedChannels: (channels) => set({ selectedChannels: channels }), toggleChannel: (channel) => { @@ -324,6 +488,21 @@ export const useEvtxStore = create()((set, get) => ({ setFilterEventIds: (eventIds) => set({ filterEventIds: eventIds }), setFilterSearch: (search) => set({ filterSearch: search }), + setTimeWindow: (window) => set({ timeWindow: window }), + // Changing the grouping invalidates every collapse key, so the old set is discarded rather than + // left to collapse unrelated groups that happen to share a key. + setGroupBy: (fields) => set({ groupBy: fields, collapsedGroups: new Set() }), + toggleColumnVisible: (id) => + set({ columnConfig: sanitizeColumnConfig(toggleColumn(get().columnConfig, id)) }), + moveColumnBy: (id, direction) => + set({ columnConfig: sanitizeColumnConfig(moveColumn(get().columnConfig, id, direction)) }), + resetColumns: () => set({ columnConfig: defaultColumnConfig() }), + toggleGroup: (key) => { + const next = new Set(get().collapsedGroups); + if (next.has(key)) next.delete(key); + else next.add(key); + set({ collapsedGroups: next }); + }, setSortField: (field) => set({ sortField: field }), setSortDirection: (direction) => set({ sortDirection: direction }), setSelectedRecordId: (id) => set({ selectedRecordId: id }), @@ -344,6 +523,15 @@ export const useEvtxStore = create()((set, get) => ({ filterLevels: new Set(ALL_LEVELS), filterEventIds: "", filterSearch: "", + timeWindow: "24h", + // Reset with everything else. Gaps describe records that are gone, so surviving a reset + // would report a hole in a set no longer on screen, and a zone left over from a previous + // session would silently reinterpret the next one's timestamps. + coverageGaps: [], + timeZoneMode: "local", + columnConfig: defaultColumnConfig(), + groupBy: [], + collapsedGroups: new Set(), sortField: "time", sortDirection: "asc", selectedRecordId: null, @@ -357,3 +545,64 @@ listen<{ channel: string; fetched: number }>("evtx-query-progress", (event) => { loadingProgress: event.payload.fetched, }); }); + +/** + * Records arriving in batches while a query is still running. + * + * A channel can be most of a scan on its own: Security measured 286,401 of 404,769 events and 191.8 + * seconds of 267, so waiting for the reply meant three minutes of empty list. Batches are collected + * here as they arrive and drained by the query that asked for them. + * + * Keyed by channel, and the sequence numbers are kept rather than discarded. An event channel makes + * no delivery promise, so the query checks both the count and the sequence run before treating a + * channel as complete. A batch that never arrived would otherwise be indistinguishable from events + * that do not exist, which is the failure this workspace exists to avoid. + */ +const pendingBatches = new Map }>(); + +listen<{ channel: string; sequence: number; records: EvtxRecord[] }>( + "evtx-record-batch", + (event) => { + const { channel, sequence, records } = event.payload; + let pending = pendingBatches.get(channel); + if (!pending) { + pending = { records: [], sequences: new Set() }; + pendingBatches.set(channel, pending); + } + // A repeated sequence is counted once. Appending it twice would inflate the tally and hide a + // batch that really is missing. + if (pending.sequences.has(sequence)) return; + pending.sequences.add(sequence); + pending.records.push(...records); + } +); + +/** Takes everything received for `channel` so far, and reports whether it is contiguous. */ +export function drainStreamedRecords(channel: string): { + records: EvtxRecord[]; + missingSequences: number[]; +} { + const pending = pendingBatches.get(channel); + pendingBatches.delete(channel); + if (!pending) return { records: [], missingSequences: [] }; + + // Batches are numbered from zero, so any number below the highest that never arrived is a batch + // whose records are simply absent. Reduced rather than spread: one argument per batch throws + // RangeError once a channel produces more batches than the engine accepts as arguments, which a + // reduced fetch batch makes reachable. + let highest = 0; + for (const sequence of pending.sequences) { + if (sequence > highest) highest = sequence; + } + const missingSequences: number[] = []; + for (let i = 0; i < highest; i++) { + if (!pending.sequences.has(i)) missingSequences.push(i); + } + return { records: pending.records, missingSequences }; +} + +/** Discards anything buffered for channels a query is about to start, so a retry cannot double up. */ +export function resetStreamedRecords(channels: string[]) { + for (const channel of channels) pendingBatches.delete(channel); +} + diff --git a/src/workspaces/event-log/evtx-time.test.ts b/src/workspaces/event-log/evtx-time.test.ts new file mode 100644 index 000000000..40a2eb7be --- /dev/null +++ b/src/workspaces/event-log/evtx-time.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from "vitest"; +import { eventDateKey, formatEventTime, timeZoneLabel } from "./evtx-time"; + +// 2026-02-10T16:36:04.390Z +const EPOCH = Date.UTC(2026, 1, 10, 16, 36, 4, 390); + +describe("formatEventTime", () => { + it("shows UTC when asked for UTC, regardless of where the machine is", () => { + expect(formatEventTime(EPOCH, "utc")).toBe("2026-02-10 16:36:04.390"); + }); + + it("shows the same instant in local time", () => { + // Compared against the platform's own conversion rather than a hardcoded string, so the test + // is not pinned to the timezone the suite happens to run in. + const local = new Date(EPOCH); + expect(formatEventTime(EPOCH, "local")).toBe( + `${local.getFullYear()}-${String(local.getMonth() + 1).padStart(2, "0")}-` + + `${String(local.getDate()).padStart(2, "0")} ` + + `${String(local.getHours()).padStart(2, "0")}:` + + `${String(local.getMinutes()).padStart(2, "0")}:` + + `${String(local.getSeconds()).padStart(2, "0")}.390` + ); + }); + + it("keeps the sub-millisecond digits Windows wrote", () => { + // Ordering two events inside the same millisecond is exactly what this tool is for, so the + // precision the source supplied must survive. + expect(formatEventTime(EPOCH, "utc", "2026-02-10T16:36:04.390987Z")).toBe( + "2026-02-10 16:36:04.390987" + ); + }); + + it("invents no precision the source did not have", () => { + expect(formatEventTime(EPOCH, "utc", "2026-02-10T16:36:04Z")).toBe( + "2026-02-10 16:36:04.390" + ); + expect(formatEventTime(EPOCH, "utc", "2026-02-10T16:36:04.390Z")).toBe( + "2026-02-10 16:36:04.390" + ); + }); + + it("takes the displayed value from the epoch, not the string", () => { + // The epoch is what rows are sorted by. Rendering from the string instead would let a row + // display a time that disagrees with the position it was sorted into. + expect(formatEventTime(EPOCH, "utc", "1999-01-01T00:00:00.000123Z")).toBe( + "2026-02-10 16:36:04.390123" + ); + }); + + it("pads every field so times stay column-aligned", () => { + const early = Date.UTC(2026, 0, 2, 3, 4, 5, 6); + expect(formatEventTime(early, "utc")).toBe("2026-01-02 03:04:05.006"); + }); +}); + +describe("eventDateKey", () => { + it("buckets by the same zone the time is shown in", () => { + // An event must not appear under a day that disagrees with the timestamp printed beside it. + expect(eventDateKey(EPOCH, "utc")).toBe("2026-02-10"); + const local = new Date(EPOCH); + expect(eventDateKey(EPOCH, "local")).toBe( + `${local.getFullYear()}-${String(local.getMonth() + 1).padStart(2, "0")}-` + + `${String(local.getDate()).padStart(2, "0")}` + ); + }); + + it("puts an instant near midnight UTC on the UTC day", () => { + expect(eventDateKey(Date.UTC(2026, 1, 10, 23, 59, 59), "utc")).toBe("2026-02-10"); + expect(eventDateKey(Date.UTC(2026, 1, 11, 0, 0, 1), "utc")).toBe("2026-02-11"); + }); +}); + +describe("timeZoneLabel", () => { + it("labels UTC plainly", () => { + expect(timeZoneLabel("utc")).toBe("UTC"); + }); + + it("labels local with its actual offset, not the word local", () => { + // A screenshot or a pasted note has to still say which clock it was. + const label = timeZoneLabel("local", EPOCH); + expect(label).toMatch(/^UTC[+-]\d{2}:\d{2}$/); + }); +}); diff --git a/src/workspaces/event-log/evtx-time.ts b/src/workspaces/event-log/evtx-time.ts new file mode 100644 index 000000000..52f0282f4 --- /dev/null +++ b/src/workspaces/event-log/evtx-time.ts @@ -0,0 +1,102 @@ +/** + * Formatting event timestamps in a stated zone. + * + * Three places rendered event times three different ways: the list column printed the raw ISO + * string Windows wrote, which is UTC; the unified timeline printed local time; and day grouping + * bucketed by local date. So the same event showed a different clock depending on where you looked + * at it, and nothing on screen said which zone any of them was. An admin correlating an event to + * the time a user reported a problem would be silently hours out. + * + * Everything now goes through here, and the zone is always labelled. + * + * Windows writes sub-millisecond precision (`.390987`) that an epoch-milliseconds value cannot + * hold. Those digits are read back off the original string rather than dropped, because ordering + * two events inside the same millisecond is exactly the kind of question this tool exists to + * answer. + */ + +/** Which clock event times are shown in. */ +export type EvtxTimeZoneMode = "local" | "utc"; + +const pad = (value: number, width = 2) => String(value).padStart(width, "0"); + +/** + * The fractional-seconds digits Windows wrote, beyond the three an epoch value keeps. + * + * Returns an empty string when the source had no more precision to offer, so nothing is invented. + */ +function extraPrecision(isoTimestamp: string | undefined): string { + if (!isoTimestamp) return ""; + const match = /\.(\d+)/.exec(isoTimestamp); + if (!match) return ""; + return match[1].slice(3); +} + +/** + * Formats an event time as `YYYY-MM-DD HH:MM:SS.mmm`. + * + * `isoTimestamp` is optional and only supplies precision past milliseconds. The value shown always + * comes from `epochMs`, so a row cannot disagree with the order it was sorted into. + */ +export function formatEventTime( + epochMs: number, + mode: EvtxTimeZoneMode, + isoTimestamp?: string +): string { + const date = new Date(epochMs); + const parts = + mode === "utc" + ? { + year: date.getUTCFullYear(), + month: date.getUTCMonth() + 1, + day: date.getUTCDate(), + hours: date.getUTCHours(), + minutes: date.getUTCMinutes(), + seconds: date.getUTCSeconds(), + ms: date.getUTCMilliseconds(), + } + : { + year: date.getFullYear(), + month: date.getMonth() + 1, + day: date.getDate(), + hours: date.getHours(), + minutes: date.getMinutes(), + seconds: date.getSeconds(), + ms: date.getMilliseconds(), + }; + + return ( + `${parts.year}-${pad(parts.month)}-${pad(parts.day)} ` + + `${pad(parts.hours)}:${pad(parts.minutes)}:${pad(parts.seconds)}.` + + `${pad(parts.ms, 3)}${extraPrecision(isoTimestamp)}` + ); +} + +/** + * The date an event falls on, for grouping. + * + * Uses the same zone as the displayed time, so an event never appears under a day that disagrees + * with the timestamp printed next to it. + */ +export function eventDateKey(epochMs: number, mode: EvtxTimeZoneMode): string { + const date = new Date(epochMs); + return mode === "utc" + ? `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())}` + : `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`; +} + +/** + * How the current zone is labelled in the UI. + * + * A time with no zone next to it is a time the reader has to guess at, which is the problem this + * module exists to remove. The local label carries the actual offset rather than the word "local", + * so a screenshot or an exported note still says which clock it was. + */ +export function timeZoneLabel(mode: EvtxTimeZoneMode, epochMs = Date.now()): string { + if (mode === "utc") return "UTC"; + // getTimezoneOffset is minutes *behind* UTC, so the sign is inverted for display. + const offsetMinutes = -new Date(epochMs).getTimezoneOffset(); + const sign = offsetMinutes < 0 ? "-" : "+"; + const absolute = Math.abs(offsetMinutes); + return `UTC${sign}${pad(Math.floor(absolute / 60))}:${pad(absolute % 60)}`; +} diff --git a/src/workspaces/event-log/types.ts b/src/workspaces/event-log/types.ts index 50e7b317b..2ccca949f 100644 --- a/src/workspaces/event-log/types.ts +++ b/src/workspaces/event-log/types.ts @@ -12,6 +12,27 @@ export interface EvtxRecord { eventData: EvtxField[]; rawXml: string; sourceLabel: string; + /** Provider-defined task grouping, absent when the event declares none. */ + task?: number | null; + /** Operation within the task. */ + opcode?: number | null; + /** Emitting process. */ + processId?: number | null; + /** Emitting thread. */ + threadId?: number | null; + /** Raw security identifier; not resolved to an account name. */ + userSid?: string | null; + /** Keyword bitmask as written by the provider. */ + keywords?: string | null; + /** Columns produced by an EvtxECmd map; empty when no map covers this event type. */ + mapped?: EvtxMappedColumn[]; +} + +export interface EvtxMappedColumn { + property: string; + text: string; + /** False when the map referenced a field this event did not carry. */ + complete: boolean; } export interface EvtxField { @@ -34,3 +55,43 @@ export interface EvtxParseResult { parseErrors: number; errorMessages: string[]; } + +/** + * How far back a live query reaches. + * + * Applied by the Event Log service as an XPath predicate, so events outside the window are never + * fetched or rendered. This is the difference between a bounded query and a walk of every channel: + * FullEventLogView filters time client-side, which is why its seven-day default is slow. + */ +export type EvtxTimeWindow = "1h" | "24h" | "7d" | "30d" | "all"; + +export const EVTX_TIME_WINDOW_MS: Record, number> = { + "1h": 60 * 60 * 1000, + "24h": 24 * 60 * 60 * 1000, + "7d": 7 * 24 * 60 * 60 * 1000, + "30d": 30 * 24 * 60 * 60 * 1000, +}; + +export const EVTX_TIME_WINDOW_LABELS: Record = { + "1h": "Last hour", + "24h": "Last 24 hours", + "7d": "Last 7 days", + "30d": "Last 30 days", + all: "All time", +}; + +/** + * The subset of the backend's query filter this workspace currently sends. + * + * Deliberately not the whole contract. `cmtraceopen_parser::event_query::EventQueryFilter` also + * carries `eventIds`, `eventIdMode`, `providerMode` and `keywords`, and its `TimeWindow` has a + * `between` variant as well as `last`. The Rust struct is `#[serde(default)]`, so omitting them + * deserializes to defaults and this works; what is absent is UI for them, not backend support. + * + * Named as a subset so nobody reads a missing field here as a capability the backend lacks. + */ +export interface EventQueryFilterSubset { + time?: { kind: "last"; milliseconds: number }; + levels?: number[]; + providers?: string[]; +} diff --git a/src/workspaces/event-log/unified-timeline.test.ts b/src/workspaces/event-log/unified-timeline.test.ts new file mode 100644 index 000000000..1ff7629c4 --- /dev/null +++ b/src/workspaces/event-log/unified-timeline.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it } from "vitest"; +import { + isEventOrigin, + originDetail, + originLabel, + timelineCounts, + unplacedSummary, + type TimelineOrigin, + type UnifiedTimeline, +} from "./unified-timeline"; + +const logOrigin: TimelineOrigin = { + kind: "log", + file: "C:\\ProgramData\\Microsoft\\IntuneManagementExtension\\Logs\\IntuneManagementExtension.log", + component: "IME", + line: 42, +}; + +const eventOrigin: TimelineOrigin = { + kind: "event", + channel: "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider/Admin", + provider: "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider", + eventId: 76, + recordId: 1234, +}; + +function timeline(partial: Partial = {}): UnifiedTimeline { + return { items: [], unplaced: [], ...partial }; +} + +describe("originLabel", () => { + it("shows the log file name and component, not the whole path", () => { + expect(originLabel(logOrigin)).toBe("IntuneManagementExtension.log [IME]"); + }); + + it("omits the component when the format has none", () => { + expect(originLabel({ ...logOrigin, component: null })).toBe( + "IntuneManagementExtension.log" + ); + }); + + it("shows the channel leaf and event id, not the Microsoft-Windows prefix", () => { + // That prefix is on nearly every channel and distinguishes nothing in a narrow column. + expect(originLabel(eventOrigin)).toBe("Admin (76)"); + }); + + it("falls back to the whole value when there is no separator", () => { + expect(originLabel({ ...eventOrigin, channel: "Security" })).toBe("Security (76)"); + expect(originLabel({ ...logOrigin, file: "app.log", component: null })).toBe("app.log"); + }); +}); + +describe("originDetail", () => { + it("gives the full path and line for a log", () => { + expect(originDetail(logOrigin)).toContain("IntuneManagementExtension.log:42"); + expect(originDetail(logOrigin)).toContain("(IME)"); + }); + + it("gives channel, provider, event and record for an event", () => { + // All four, as the name promises. Asserting only two let a change that dropped the channel or + // the provider from the detail line pass. + const detail = originDetail(eventOrigin); + expect(detail).toContain(eventOrigin.channel); + expect(detail).toContain(eventOrigin.provider); + expect(detail).toContain("event 76"); + expect(detail).toContain("record 1234"); + }); +}); + +describe("isEventOrigin", () => { + it("distinguishes the two sources", () => { + expect(isEventOrigin(eventOrigin)).toBe(true); + expect(isEventOrigin(logOrigin)).toBe(false); + }); +}); + +describe("unplacedSummary", () => { + it("returns null when nothing was dropped", () => { + // A reassuring "0 items" invites no attention, so the caller hides the notice entirely. + expect(unplacedSummary(timeline())).toBeNull(); + }); + + it("counts both sources and says why", () => { + const summary = unplacedSummary( + timeline({ + unplaced: [ + { origin: logOrigin, reason: "missingTimestamp" }, + { origin: logOrigin, reason: "missingTimestamp" }, + { origin: eventOrigin, reason: "missingTimestamp" }, + ], + }) + ); + expect(summary).toBe("2 log lines and 1 event could not be placed: no timestamp"); + }); + + it("uses singular wording for a single item", () => { + expect( + unplacedSummary(timeline({ unplaced: [{ origin: logOrigin, reason: "missingTimestamp" }] })) + ).toBe("1 log line could not be placed: no timestamp"); + }); + + it("mentions only the source that actually contributed", () => { + const summary = unplacedSummary( + timeline({ unplaced: [{ origin: eventOrigin, reason: "missingTimestamp" }] }) + ); + expect(summary).toBe("1 event could not be placed: no timestamp"); + }); +}); + +describe("timelineCounts", () => { + it("separates events from log lines", () => { + const counts = timelineCounts( + timeline({ + items: [ + { timestampMs: 1, severity: "info", message: "a", origin: logOrigin }, + { timestampMs: 2, severity: "error", message: "b", origin: eventOrigin }, + { timestampMs: 3, severity: "info", message: "c", origin: logOrigin }, + ], + unplaced: [{ origin: logOrigin, reason: "missingTimestamp" }], + }) + ); + expect(counts).toEqual({ logs: 2, events: 1, unplaced: 1 }); + }); + + it("counts an empty timeline as zero everywhere", () => { + expect(timelineCounts(timeline())).toEqual({ logs: 0, events: 0, unplaced: 0 }); + }); +}); diff --git a/src/workspaces/event-log/unified-timeline.ts b/src/workspaces/event-log/unified-timeline.ts new file mode 100644 index 000000000..d479c5881 --- /dev/null +++ b/src/workspaces/event-log/unified-timeline.ts @@ -0,0 +1,110 @@ +/** + * Frontend model for the unified timeline. + * + * Mirrors `cmtraceopen_parser::unified_timeline`. The merge itself happens in Rust; this side + * handles presentation, which mostly means deciding what to show about items that could not be + * placed. + */ + +export type TimelineSeverity = + | "verbose" + | "info" + | "warning" + | "error" + | "critical"; + +export type TimelineOrigin = + | { kind: "log"; file: string; component: string | null; line: number } + | { + kind: "event"; + channel: string; + provider: string; + eventId: number; + recordId: number; + }; + +export interface TimelineItem { + timestampMs: number; + severity: TimelineSeverity; + message: string; + origin: TimelineOrigin; +} + +export interface UnplacedItem { + origin: TimelineOrigin; + reason: "missingTimestamp"; +} + +export interface UnifiedTimeline { + items: TimelineItem[]; + unplaced: UnplacedItem[]; +} + +export const TIMELINE_SEVERITY_RANK: Record = { + verbose: 0, + info: 1, + warning: 2, + error: 3, + critical: 4, +}; + +/** Short label for an item's source, for a column narrow enough to scan. */ +export function originLabel(origin: TimelineOrigin): string { + if (origin.kind === "log") { + // The file name alone; the full path is available as a tooltip and is far too long to scan. + const name = origin.file.split(/[\\/]/).pop() || origin.file; + return origin.component ? `${name} [${origin.component}]` : name; + } + // The leaf of the channel path, since the Microsoft-Windows- prefix is on nearly every channel + // and carries no distinguishing information in a narrow column. + const leaf = origin.channel.split("/").pop() || origin.channel; + return `${leaf} (${origin.eventId})`; +} + +/** Full source description, for a tooltip. */ +export function originDetail(origin: TimelineOrigin): string { + if (origin.kind === "log") { + return `${origin.file}:${origin.line}${origin.component ? ` (${origin.component})` : ""}`; + } + return `${origin.channel} / ${origin.provider} / event ${origin.eventId} / record ${origin.recordId}`; +} + +/** True when the item came from a Windows event rather than a text log. */ +export function isEventOrigin(origin: TimelineOrigin): boolean { + return origin.kind === "event"; +} + +/** + * Human-readable summary of what could not be placed. + * + * Returns null when nothing was dropped, so the caller can hide the notice entirely rather than + * showing a reassuring "0 items" that invites no attention. + */ +export function unplacedSummary(timeline: UnifiedTimeline): string | null { + const total = timeline.unplaced.length; + if (total === 0) return null; + + const logs = timeline.unplaced.filter((item) => item.origin.kind === "log").length; + const events = total - logs; + + const parts: string[] = []; + if (logs > 0) parts.push(`${logs} log ${logs === 1 ? "line" : "lines"}`); + if (events > 0) parts.push(`${events} ${events === 1 ? "event" : "events"}`); + + return `${parts.join(" and ")} could not be placed: no timestamp`; +} + +/** Counts by source kind, for a coverage readout. */ +export function timelineCounts(timeline: UnifiedTimeline): { + logs: number; + events: number; + unplaced: number; +} { + let logs = 0; + let events = 0; + for (const item of timeline.items) { + if (item.origin.kind === "event") events += 1; + else logs += 1; + } + return { logs, events, unplaced: timeline.unplaced.length }; +}