From 8dd5c9f36cb231fd5a7b8f6a7a541e4c930ab387 Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 9 Aug 2026 12:41:46 -0400 Subject: [PATCH 01/85] feat(parser): add the EvtxECmd map engine Every event ID carries a different EventData shape, which is why event logs resist tabular display. EvtxECmd solved this with community maps that project each event's fields into a fixed set of columns. Implementing that schema means the existing upstream corpus works unmodified, and maps written here work in EvtxECmd and Timeline Explorer. The grammar was measured, not assumed. Across all 468 upstream maps there are 1,837 value expressions in six shapes: named EventData (1,441), nested UserData (204), bare Data (176), System paths including Correlation/@ActivityID (12), indexed Data (3), and the EventData container (1). So this implements an absolute element path with optional attribute-equality or 1-based index predicates and an optional trailing attribute selector, rather than a general XPath engine the corpus does not justify. Two boundaries keep the crate pure and wasm32-compatible, and both add zero dependencies: - No XML crate. Callers convert what they already hold into EventNode. - No YAML crate. The schema derives serde::Deserialize, so format-specific loading belongs in the host layer. Upstream maps are YAML; the vendored fixtures are the same maps as JSON. Resolution is non-fatal by design. Maps are written against a provider's superset of fields, so an individual event legitimately omits some. A missing field is reported on MappedValue::unresolved and leaves its %placeholder% visible rather than blanking it, which would present a partial value as a complete one. A malformed path is a defect in the map file, so it is reported separately from a missing field. Two corpus details are handled deliberately: four upstream maps spell the target "Username" rather than "UserName", so Property parsing is ASCII case-insensitive; and unknown targets are preserved as MapProperty::Other so a map contributed against a newer schema is not silently discarded. Fixtures are unmodified upstream maps (MIT, attributed in the fixtures README) converted from YAML to JSON, so the engine is tested against the real schema rather than invented examples. Open question recorded in path.rs: whether EvtxECmd joins repeated unnamed elements or takes the first. This takes the first; to be confirmed against the tool on the lab host. Refs #539. Gates: 2,203 tests pass, clippy -D warnings clean, rustfmt clean, and cargo check --target wasm32-unknown-unknown still succeeds. Co-Authored-By: Claude Opus 5 --- .../cmtraceopen-parser/src/eventmap/apply.rs | 337 ++++++++++++++++++ crates/cmtraceopen-parser/src/eventmap/mod.rs | 210 +++++++++++ .../cmtraceopen-parser/src/eventmap/model.rs | 216 +++++++++++ .../cmtraceopen-parser/src/eventmap/node.rs | 112 ++++++ .../cmtraceopen-parser/src/eventmap/path.rs | 331 +++++++++++++++++ crates/cmtraceopen-parser/src/lib.rs | 1 + .../tests/eventmap_corpus.rs | 209 +++++++++++ .../tests/fixtures/eventmap/README.md | 14 + .../fixtures/eventmap/ntfs-146-lookups.json | 96 +++++ .../fixtures/eventmap/security-4624.json | 101 ++++++ .../fixtures/eventmap/shell-core-9701.json | 19 + 11 files changed, 1646 insertions(+) create mode 100644 crates/cmtraceopen-parser/src/eventmap/apply.rs create mode 100644 crates/cmtraceopen-parser/src/eventmap/mod.rs create mode 100644 crates/cmtraceopen-parser/src/eventmap/model.rs create mode 100644 crates/cmtraceopen-parser/src/eventmap/node.rs create mode 100644 crates/cmtraceopen-parser/src/eventmap/path.rs create mode 100644 crates/cmtraceopen-parser/tests/eventmap_corpus.rs create mode 100644 crates/cmtraceopen-parser/tests/fixtures/eventmap/README.md create mode 100644 crates/cmtraceopen-parser/tests/fixtures/eventmap/ntfs-146-lookups.json create mode 100644 crates/cmtraceopen-parser/tests/fixtures/eventmap/security-4624.json create mode 100644 crates/cmtraceopen-parser/tests/fixtures/eventmap/shell-core-9701.json diff --git a/crates/cmtraceopen-parser/src/eventmap/apply.rs b/crates/cmtraceopen-parser/src/eventmap/apply.rs new file mode 100644 index 000000000..6c2aa2c0c --- /dev/null +++ b/crates/cmtraceopen-parser/src/eventmap/apply.rs @@ -0,0 +1,337 @@ +//! Applying a map to an event. +//! +//! Resolution is deliberately non-fatal. Maps are written against a provider's superset of +//! fields, so an individual event legitimately omits some of them. A missing field is a coverage +//! state, reported on [`MappedValue::unresolved`], never an error and never silently blank. + +use super::model::{EventMap, MapEntry, MapProperty}; +use super::node::EventNode; +use super::path::ValuePath; + +/// One normalized column produced from an event. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MappedValue { + /// The column this fills. + pub property: MapProperty, + /// The rendered text, with every resolved placeholder substituted. + pub text: String, + /// Variables whose paths matched nothing, in template order. + /// + /// Non-empty means `text` still contains their raw `%Name%` placeholders. Callers decide + /// whether to show the column, hide it, or flag it; the engine refuses to guess by blanking + /// the placeholder, which would present a partial value as a complete one. + pub unresolved: Vec, +} + +impl MappedValue { + /// True when every placeholder in the template resolved. + pub fn is_complete(&self) -> bool { + self.unresolved.is_empty() + } +} + +/// The full result of applying a map to one event. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct MappedEvent { + /// Columns in map order. + pub values: Vec, + /// Path expressions that failed to parse, as `(variable, expression)`. + /// + /// A malformed expression is a defect in the map file rather than in the event, so it is + /// surfaced separately instead of being reported as a missing field. + pub invalid_paths: Vec<(String, String)>, +} + +impl MappedEvent { + /// Returns the text written to `property`, if the map produced it. + pub fn value_for(&self, property: &MapProperty) -> Option<&str> { + self.values + .iter() + .find(|value| &value.property == property) + .map(|value| value.text.as_str()) + } +} + +/// Applies every entry of `map` to `event`, which must be the `Event` element. +pub fn apply_map(map: &EventMap, event: &EventNode) -> MappedEvent { + let mut result = MappedEvent::default(); + for entry in &map.maps { + result + .values + .push(apply_entry(map, entry, event, &mut result.invalid_paths)); + } + result +} + +fn apply_entry( + map: &EventMap, + entry: &MapEntry, + event: &EventNode, + invalid_paths: &mut Vec<(String, String)>, +) -> MappedValue { + let mut text = entry.property_value.clone(); + let mut unresolved = Vec::new(); + + for binding in &entry.values { + let placeholder = format!("%{}%", binding.name); + if !text.contains(&placeholder) { + continue; + } + + let resolved = match ValuePath::parse(&binding.value) { + Ok(path) => path.evaluate(event).map(str::to_string), + Err(_) => { + let defect = (binding.name.clone(), binding.value.clone()); + if !invalid_paths.contains(&defect) { + invalid_paths.push(defect); + } + None + } + }; + + let translated = resolved.and_then(|raw| match map.lookup_for(&binding.name) { + Some(lookup) => lookup.translate(&raw), + None => Some(raw), + }); + + match translated { + Some(value) => text = text.replace(&placeholder, &value), + None => unresolved.push(binding.name.clone()), + } + } + + // A template may reference a variable the map never binds. That is still an unresolved + // placeholder from the reader's point of view, so report it the same way. + for name in remaining_placeholders(&text) { + if !unresolved.contains(&name) { + unresolved.push(name); + } + } + + MappedValue { + property: entry.property.clone(), + text, + unresolved, + } +} + +fn remaining_placeholders(text: &str) -> Vec { + let mut names = Vec::new(); + let mut rest = text; + while let Some(start) = rest.find('%') { + let after = &rest[start + 1..]; + let Some(end) = after.find('%') else { break }; + let name = &after[..end]; + if !name.is_empty() && !name.contains(char::is_whitespace) { + names.push(name.to_string()); + } + rest = &after[end + 1..]; + } + names +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::eventmap::model::{Lookup, MapEntry, ValueBinding}; + use std::collections::BTreeMap; + + fn event() -> EventNode { + EventNode::new("Event").with_child( + EventNode::new("EventData") + .with_child( + EventNode::new("Data") + .with_attribute("Name", "SubjectUserName") + .with_text("adam"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "SubjectDomainName") + .with_text("TEST"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "BusType") + .with_text("7"), + ), + ) + } + + fn binding(name: &str, field: &str) -> ValueBinding { + ValueBinding { + name: name.to_string(), + value: format!(r#"/Event/EventData/Data[@Name="{field}"]"#), + } + } + + fn map_with(entries: Vec, lookups: Vec) -> EventMap { + EventMap { + author: None, + description: None, + event_id: 4624, + channel: "Security".to_string(), + provider: "Microsoft-Windows-Security-Auditing".to_string(), + maps: entries, + lookups, + } + } + + #[test] + fn substitutes_multiple_placeholders_in_one_template() { + let map = map_with( + vec![MapEntry { + property: MapProperty::UserName, + property_value: "%domain%\\%user%".to_string(), + values: vec![ + binding("domain", "SubjectDomainName"), + binding("user", "SubjectUserName"), + ], + }], + vec![], + ); + + let mapped = apply_map(&map, &event()); + assert_eq!(mapped.value_for(&MapProperty::UserName), Some("TEST\\adam")); + assert!(mapped.values[0].is_complete()); + } + + #[test] + fn missing_field_is_reported_and_leaves_the_placeholder_visible() { + let map = map_with( + vec![MapEntry { + property: MapProperty::PayloadData(1), + property_value: "LogonType %LogonType%".to_string(), + values: vec![binding("LogonType", "LogonType")], + }], + vec![], + ); + + let mapped = apply_map(&map, &event()); + let value = &mapped.values[0]; + assert_eq!(value.unresolved, vec!["LogonType".to_string()]); + assert!(!value.is_complete()); + assert_eq!(value.text, "LogonType %LogonType%"); + } + + #[test] + fn lookup_translates_a_bound_variable() { + let map = map_with( + vec![MapEntry { + property: MapProperty::PayloadData(1), + property_value: "Bus: %BusType%".to_string(), + values: vec![binding("BusType", "BusType")], + }], + vec![Lookup { + name: "BusType".to_string(), + default: Some("Unknown code".to_string()), + values: BTreeMap::from([("7".to_string(), "USB".to_string())]), + }], + ); + + let mapped = apply_map(&map, &event()); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("Bus: USB") + ); + } + + #[test] + fn lookup_default_applies_to_an_unknown_code() { + let map = map_with( + vec![MapEntry { + property: MapProperty::PayloadData(1), + property_value: "Bus: %BusType%".to_string(), + values: vec![binding("BusType", "BusType")], + }], + vec![Lookup { + name: "BusType".to_string(), + default: Some("Unknown code".to_string()), + values: BTreeMap::new(), + }], + ); + + let mapped = apply_map(&map, &event()); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("Bus: Unknown code") + ); + } + + #[test] + fn a_template_with_no_placeholders_is_emitted_verbatim() { + let map = map_with( + vec![MapEntry { + property: MapProperty::PayloadData(1), + property_value: "Screen saver invoked".to_string(), + values: vec![], + }], + vec![], + ); + + let mapped = apply_map(&map, &event()); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("Screen saver invoked") + ); + assert!(mapped.values[0].is_complete()); + } + + #[test] + fn a_placeholder_with_no_binding_is_reported_unresolved() { + let map = map_with( + vec![MapEntry { + property: MapProperty::PayloadData(1), + property_value: "Value %NeverBound%".to_string(), + values: vec![], + }], + vec![], + ); + + let mapped = apply_map(&map, &event()); + assert_eq!(mapped.values[0].unresolved, vec!["NeverBound".to_string()]); + } + + #[test] + fn a_malformed_path_is_a_map_defect_not_a_missing_field() { + let map = map_with( + vec![MapEntry { + property: MapProperty::PayloadData(1), + property_value: "%broken%".to_string(), + values: vec![ValueBinding { + name: "broken".to_string(), + value: "EventData/Data".to_string(), + }], + }], + vec![], + ); + + let mapped = apply_map(&map, &event()); + assert_eq!( + mapped.invalid_paths, + vec![("broken".to_string(), "EventData/Data".to_string())] + ); + assert_eq!(mapped.values[0].unresolved, vec!["broken".to_string()]); + } + + #[test] + fn an_unused_binding_does_not_affect_the_result() { + let map = map_with( + vec![MapEntry { + property: MapProperty::PayloadData(1), + property_value: "User %user%".to_string(), + values: vec![ + binding("user", "SubjectUserName"), + binding("unused", "DoesNotExist"), + ], + }], + vec![], + ); + + let mapped = apply_map(&map, &event()); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("User adam") + ); + assert!(mapped.values[0].is_complete()); + } +} diff --git a/crates/cmtraceopen-parser/src/eventmap/mod.rs b/crates/cmtraceopen-parser/src/eventmap/mod.rs new file mode 100644 index 000000000..e9f7b9f9e --- /dev/null +++ b/crates/cmtraceopen-parser/src/eventmap/mod.rs @@ -0,0 +1,210 @@ +//! Normalizing Windows event data into stable columns. +//! +//! Every event ID carries a different `EventData` shape, which is why event logs resist tabular +//! display. EvtxECmd solved this with community "maps" that project each event's fields into a +//! fixed set of columns. This module implements that schema so the existing corpus of upstream +//! maps works unmodified, and so maps written here work in EvtxECmd and Timeline Explorer. +//! +//! Two boundaries keep this crate pure and wasm32-compatible: +//! +//! - **No XML dependency.** Callers convert whatever they hold into [`EventNode`]. +//! - **No YAML dependency.** The schema derives `serde::Deserialize`, so the host layer picks the +//! format. Upstream maps are YAML; the fixtures here are the same maps as JSON. +//! +//! ``` +//! use cmtraceopen_parser::eventmap::{apply_map, EventMap, EventNode, MapProperty}; +//! +//! let map: EventMap = serde_json::from_str(r#"{ +//! "EventId": 9701, +//! "Channel": "Microsoft-Windows-Shell-Core/Operational", +//! "Provider": "Microsoft-Windows-Shell-Core", +//! "Maps": [{ +//! "Property": "PayloadData1", +//! "PropertyValue": "%PayloadData1%", +//! "Values": [{ "Name": "PayloadData1", "Value": "/Event/EventData/Data" }] +//! }] +//! }"#).unwrap(); +//! +//! let event = EventNode::new("Event").with_child( +//! EventNode::new("EventData").with_child(EventNode::new("Data").with_text("RunOnceEx")), +//! ); +//! +//! let mapped = apply_map(&map, &event); +//! assert_eq!(mapped.value_for(&MapProperty::PayloadData(1)), Some("RunOnceEx")); +//! ``` + +mod apply; +mod model; +mod node; +mod path; + +pub use apply::{apply_map, MappedEvent, MappedValue}; +pub use model::{EventMap, Lookup, MapEntry, MapProperty, ValueBinding}; +pub use node::EventNode; +pub use path::{PathError, Predicate, Step, ValuePath}; + +use std::collections::HashMap; + +/// Identity of a map: channel, provider, and event ID. +/// +/// The file name is not part of identity upstream, and neither is it here. Channel and provider +/// are compared ASCII case-insensitively, because a case mismatch would silently drop the mapping +/// for every event of that type rather than fail loudly. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct MapKey { + channel: String, + provider: String, + event_id: u32, +} + +impl MapKey { + fn new(channel: &str, provider: &str, event_id: u32) -> Self { + Self { + channel: channel.trim().to_ascii_lowercase(), + provider: provider.trim().to_ascii_lowercase(), + event_id, + } + } +} + +/// A set of maps, resolved by channel, provider, and event ID. +#[derive(Debug, Clone, Default)] +pub struct MapRegistry { + maps: HashMap, +} + +impl MapRegistry { + /// Creates an empty registry. + pub fn new() -> Self { + Self::default() + } + + /// Inserts `map`, returning any map it replaced. + /// + /// Upstream loads maps in alphabetical file order so that a `1_`-prefixed copy overrides the + /// original. Load order is the caller's concern; last insert wins here. + pub fn insert(&mut self, map: EventMap) -> Option { + let key = MapKey::new(&map.channel, &map.provider, map.event_id); + self.maps.insert(key, map) + } + + /// Finds the map for an event, if one exists. + pub fn find(&self, channel: &str, provider: &str, event_id: u32) -> Option<&EventMap> { + self.maps.get(&MapKey::new(channel, provider, event_id)) + } + + /// Number of maps held. + pub fn len(&self) -> usize { + self.maps.len() + } + + /// True when no maps are held. + pub fn is_empty(&self) -> bool { + self.maps.is_empty() + } + + /// Applies the matching map to `event`, if one is registered. + pub fn apply( + &self, + channel: &str, + provider: &str, + event_id: u32, + event: &EventNode, + ) -> Option { + self.find(channel, provider, event_id) + .map(|map| apply_map(map, event)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn map(channel: &str, provider: &str, event_id: u32) -> EventMap { + EventMap { + author: None, + description: None, + event_id, + channel: channel.to_string(), + provider: provider.to_string(), + maps: Vec::new(), + lookups: Vec::new(), + } + } + + #[test] + fn resolves_by_channel_provider_and_event_id() { + let mut registry = MapRegistry::new(); + registry.insert(map("Security", "Microsoft-Windows-Security-Auditing", 4624)); + + assert!(registry + .find("Security", "Microsoft-Windows-Security-Auditing", 4624) + .is_some()); + assert!(registry + .find("Security", "Microsoft-Windows-Security-Auditing", 4625) + .is_none()); + assert!(registry + .find("System", "Microsoft-Windows-Security-Auditing", 4624) + .is_none()); + } + + #[test] + fn channel_and_provider_matching_is_case_insensitive() { + let mut registry = MapRegistry::new(); + registry.insert(map( + "Microsoft-Windows-Shell-Core/Operational", + "Microsoft-Windows-Shell-Core", + 9701, + )); + + assert!(registry + .find( + "microsoft-windows-shell-core/operational", + "MICROSOFT-WINDOWS-SHELL-CORE", + 9701 + ) + .is_some()); + } + + #[test] + fn the_same_event_id_on_different_channels_stays_distinct() { + let mut registry = MapRegistry::new(); + registry.insert(map("Security", "Provider-A", 4624)); + registry.insert(map("System", "Provider-A", 4624)); + + assert_eq!(registry.len(), 2); + } + + #[test] + fn reinserting_the_same_identity_replaces_and_returns_the_previous_map() { + let mut registry = MapRegistry::new(); + let mut first = map("Security", "Provider-A", 4624); + first.description = Some("original".to_string()); + registry.insert(first); + + let mut override_map = map("Security", "Provider-A", 4624); + override_map.description = Some("override".to_string()); + let replaced = registry.insert(override_map); + + assert_eq!( + replaced.and_then(|m| m.description).as_deref(), + Some("original") + ); + assert_eq!(registry.len(), 1); + assert_eq!( + registry + .find("Security", "Provider-A", 4624) + .and_then(|m| m.description.as_deref()), + Some("override") + ); + } + + #[test] + fn an_empty_registry_maps_nothing() { + let registry = MapRegistry::new(); + assert!(registry.is_empty()); + assert!(registry + .apply("Security", "Provider-A", 4624, &EventNode::new("Event")) + .is_none()); + } +} diff --git a/crates/cmtraceopen-parser/src/eventmap/model.rs b/crates/cmtraceopen-parser/src/eventmap/model.rs new file mode 100644 index 000000000..33d72aafa --- /dev/null +++ b/crates/cmtraceopen-parser/src/eventmap/model.rs @@ -0,0 +1,216 @@ +//! The EvtxECmd map schema. +//! +//! Deliberately derives `serde::Deserialize` rather than depending on a YAML crate. Upstream maps +//! are YAML, but this crate is pure and wasm32-compatible, so format-specific loading belongs in +//! the host layer. Any serde format that produces these field names works, which keeps YAML out +//! of the parser crate and lets tests drive the real corpus through `serde_json`. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Deserializer}; + +/// A normalized output column that a map entry writes into. +/// +/// The upstream corpus is not perfectly consistent: four maps spell the target `Username` rather +/// than `UserName`, so parsing is ASCII case-insensitive. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum MapProperty { + /// The acting or target account. + UserName, + /// The remote host, typically a workstation name and address. + RemoteHost, + /// Process, command line, service, or scheduled-task detail. + ExecutableInfo, + /// One of the six generic overflow slots, numbered 1 through 6. + PayloadData(u8), + /// A target this build does not know, preserved rather than dropped so a community map + /// contributed against a newer schema is not silently discarded. + Other(String), +} + +impl MapProperty { + /// Parses a `Property` value, ASCII case-insensitively. + pub fn parse(raw: &str) -> Self { + let trimmed = raw.trim(); + if trimmed.eq_ignore_ascii_case("username") { + return Self::UserName; + } + if trimmed.eq_ignore_ascii_case("remotehost") { + return Self::RemoteHost; + } + if trimmed.eq_ignore_ascii_case("executableinfo") { + return Self::ExecutableInfo; + } + if let Some(suffix) = trimmed + .get(..11) + .filter(|prefix| prefix.eq_ignore_ascii_case("payloaddata")) + .and_then(|_| trimmed.get(11..)) + { + if let Ok(slot @ 1..=6) = suffix.parse::() { + return Self::PayloadData(slot); + } + } + Self::Other(trimmed.to_string()) + } +} + +impl<'de> Deserialize<'de> for MapProperty { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + Ok(Self::parse(&raw)) + } +} + +/// Binds a template variable to a location in the event. +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "PascalCase")] +pub struct ValueBinding { + /// Variable name, referenced as `%Name%` in the owning entry's `PropertyValue`. + pub name: String, + /// The path expression locating the value. + pub value: String, +} + +/// A translation table turning raw codes into readable text. +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "PascalCase")] +pub struct Lookup { + /// Matches the [`ValueBinding::name`] this table applies to. + pub name: String, + /// Text used when the raw value is absent from `values`. + #[serde(default)] + pub default: Option, + /// Raw value to readable text. + #[serde(default)] + pub values: BTreeMap, +} + +impl Lookup { + /// Translates `raw`, falling back to [`Lookup::default`] when it is not in the table. + pub fn translate(&self, raw: &str) -> Option { + self.values + .get(raw) + .cloned() + .or_else(|| self.default.clone()) + } +} + +/// One output column produced from an event. +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "PascalCase")] +pub struct MapEntry { + /// Which normalized column this writes. + pub property: MapProperty, + /// Template containing `%Name%` placeholders. + pub property_value: String, + /// Variables available to the template. + #[serde(default)] + pub values: Vec, +} + +/// A parsed EvtxECmd map file. +/// +/// Identity is `(channel, provider, event_id)`, not the file name, matching upstream. +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "PascalCase")] +pub struct EventMap { + /// Map author, carried through for attribution. + #[serde(default)] + pub author: Option, + /// What the mapped event means. + #[serde(default)] + pub description: Option, + /// The event ID this map applies to. + pub event_id: u32, + /// The channel this map applies to, for example `Security`. + pub channel: String, + /// The provider this map applies to. + pub provider: String, + /// Output columns, evaluated in order. + #[serde(default)] + pub maps: Vec, + /// Translation tables referenced by variable name. + #[serde(default)] + pub lookups: Vec, +} + +impl EventMap { + /// Returns the lookup table bound to `variable`, if any. + pub fn lookup_for(&self, variable: &str) -> Option<&Lookup> { + self.lookups + .iter() + .find(|lookup| lookup.name.eq_ignore_ascii_case(variable)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_known_property_targets() { + assert_eq!(MapProperty::parse("UserName"), MapProperty::UserName); + assert_eq!(MapProperty::parse("RemoteHost"), MapProperty::RemoteHost); + assert_eq!( + MapProperty::parse("ExecutableInfo"), + MapProperty::ExecutableInfo + ); + assert_eq!( + MapProperty::parse("PayloadData4"), + MapProperty::PayloadData(4) + ); + } + + #[test] + fn property_parsing_tolerates_the_corpus_case_inconsistency() { + // Four upstream maps spell this "Username"; treating that as unknown would silently drop + // the account column for those events. + assert_eq!(MapProperty::parse("Username"), MapProperty::UserName); + assert_eq!( + MapProperty::parse("payloaddata2"), + MapProperty::PayloadData(2) + ); + } + + #[test] + fn unknown_and_out_of_range_targets_are_preserved() { + assert_eq!( + MapProperty::parse("PayloadData7"), + MapProperty::Other("PayloadData7".to_string()) + ); + assert_eq!( + MapProperty::parse("SomethingNew"), + MapProperty::Other("SomethingNew".to_string()) + ); + } + + #[test] + fn property_parsing_does_not_panic_on_short_input() { + assert_eq!(MapProperty::parse(""), MapProperty::Other(String::new())); + assert_eq!( + MapProperty::parse("Pay"), + MapProperty::Other("Pay".to_string()) + ); + } + + #[test] + fn lookup_falls_back_to_default() { + let lookup = Lookup { + name: "BusType".to_string(), + default: Some("Unknown code".to_string()), + values: BTreeMap::from([("7".to_string(), "USB".to_string())]), + }; + assert_eq!(lookup.translate("7").as_deref(), Some("USB")); + assert_eq!(lookup.translate("99").as_deref(), Some("Unknown code")); + } + + #[test] + fn lookup_without_default_returns_none_for_unknown_codes() { + let lookup = Lookup { + name: "BusType".to_string(), + default: None, + values: BTreeMap::new(), + }; + assert_eq!(lookup.translate("7"), None); + } +} diff --git a/crates/cmtraceopen-parser/src/eventmap/node.rs b/crates/cmtraceopen-parser/src/eventmap/node.rs new file mode 100644 index 000000000..76502541f --- /dev/null +++ b/crates/cmtraceopen-parser/src/eventmap/node.rs @@ -0,0 +1,112 @@ +//! A minimal, XML-free event tree. +//! +//! The map engine has to read values out of a rendered Windows event, but this crate is pure +//! Rust and wasm32-compatible, so it must not depend on an XML reader. Callers convert whatever +//! they already have (rendered event XML in `src-tauri`, an `evtx` record, a test literal) into +//! [`EventNode`] and hand that across. + +/// A single element in a rendered event. +/// +/// `name` is the local element name with any namespace prefix already stripped, because event +/// XML paths in maps are written without prefixes (`/Event/EventData/Data`). +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct EventNode { + /// Local element name, for example `Data`. + pub name: String, + /// Attributes in document order, for example `("Name", "LogonType")`. + pub attributes: Vec<(String, String)>, + /// Element text content, if the element has any. + pub text: Option, + /// Child elements in document order. + pub children: Vec, +} + +impl EventNode { + /// Creates an element with no attributes, text, or children. + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + ..Self::default() + } + } + + /// Sets the element's text content. + #[must_use] + pub fn with_text(mut self, text: impl Into) -> Self { + self.text = Some(text.into()); + self + } + + /// Appends an attribute. + #[must_use] + pub fn with_attribute(mut self, name: impl Into, value: impl Into) -> Self { + self.attributes.push((name.into(), value.into())); + self + } + + /// Appends a child element. + #[must_use] + pub fn with_child(mut self, child: EventNode) -> Self { + self.children.push(child); + self + } + + /// Returns the value of `name`, compared case-sensitively as event XML declares it. + pub fn attribute(&self, name: &str) -> Option<&str> { + self.attributes + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.as_str()) + } + + /// Returns the child elements called `name`, in document order. + pub fn children_named<'a>(&'a self, name: &'a str) -> impl Iterator { + self.children.iter().filter(move |child| child.name == name) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample() -> EventNode { + EventNode::new("EventData") + .with_child( + EventNode::new("Data") + .with_attribute("Name", "SubjectUserName") + .with_text("adam"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "LogonType") + .with_text("10"), + ) + } + + #[test] + fn attribute_lookup_is_case_sensitive() { + let node = sample(); + let first = node.children.first().expect("first child"); + assert_eq!(first.attribute("Name"), Some("SubjectUserName")); + assert_eq!(first.attribute("name"), None); + } + + #[test] + fn children_named_preserves_document_order() { + let node = sample(); + let texts: Vec<_> = node + .children_named("Data") + .map(|child| child.text.as_deref().unwrap_or_default()) + .collect(); + assert_eq!(texts, vec!["adam", "10"]); + } + + #[test] + fn children_named_ignores_other_elements() { + let node = EventNode::new("Event") + .with_child(EventNode::new("System")) + .with_child(EventNode::new("EventData")); + assert_eq!(node.children_named("Data").count(), 0); + assert_eq!(node.children_named("System").count(), 1); + } +} diff --git a/crates/cmtraceopen-parser/src/eventmap/path.rs b/crates/cmtraceopen-parser/src/eventmap/path.rs new file mode 100644 index 000000000..b0c300d15 --- /dev/null +++ b/crates/cmtraceopen-parser/src/eventmap/path.rs @@ -0,0 +1,331 @@ +//! The tiny path language used by EvtxECmd map `Value` expressions. +//! +//! Map files describe where a value lives with what looks like XPath, but the corpus only uses a +//! very small subset. Measured across all 468 upstream maps (1,837 expressions): +//! +//! | Shape | Count | +//! |---|---| +//! | `/Event/EventData/Data[@Name="X"]` | 1,441 | +//! | `/Event/UserData//` | 204 | +//! | `/Event/EventData/Data` | 176 | +//! | `/Event/System/...`, including `Correlation/@ActivityID` | 12 | +//! | `/Event/EventData/Data[N]` | 3 | +//! | `/Event/EventData` | 1 | +//! +//! So this is an absolute element path where each step may carry an attribute-equality or +//! 1-based index predicate, optionally ending in an attribute selector. A general XPath engine +//! would be far more machinery than the grammar justifies. + +use thiserror::Error; + +use super::node::EventNode; + +/// A failure to parse a map `Value` expression. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum PathError { + /// The expression did not start with `/`. + #[error("value path must be absolute, starting with '/': {0}")] + NotAbsolute(String), + /// The expression had no element steps. + #[error("value path has no steps: {0}")] + Empty(String), + /// A predicate was opened but not closed, or was not understood. + #[error("value path has a malformed predicate in step '{step}': {path}")] + MalformedPredicate { path: String, step: String }, + /// An attribute selector appeared somewhere other than the final step. + #[error("value path may only select an attribute in its final step: {0}")] + MisplacedAttribute(String), +} + +/// Narrows which sibling a step selects. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Predicate { + /// `[@Name="value"]`, selecting the first sibling whose attribute matches. + AttributeEquals { name: String, value: String }, + /// `[n]`, a 1-based index across same-named siblings, as XPath numbers them. + Index(usize), +} + +/// One element step of a path. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Step { + /// Element name to match. + pub name: String, + /// Optional narrowing predicate. + pub predicate: Option, +} + +/// A parsed map `Value` expression. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ValuePath { + steps: Vec, + attribute: Option, +} + +impl ValuePath { + /// Parses an expression such as `/Event/EventData/Data[@Name="LogonType"]`. + pub fn parse(expression: &str) -> Result { + let trimmed = expression.trim(); + let Some(body) = trimmed.strip_prefix('/') else { + return Err(PathError::NotAbsolute(expression.to_string())); + }; + + let mut steps = Vec::new(); + let mut attribute = None; + let segments: Vec<&str> = body.split('/').filter(|s| !s.is_empty()).collect(); + + for (index, segment) in segments.iter().enumerate() { + if let Some(attribute_name) = segment.strip_prefix('@') { + if index + 1 != segments.len() { + return Err(PathError::MisplacedAttribute(expression.to_string())); + } + attribute = Some(attribute_name.to_string()); + continue; + } + steps.push(parse_step(segment, expression)?); + } + + if steps.is_empty() { + return Err(PathError::Empty(expression.to_string())); + } + + Ok(Self { steps, attribute }) + } + + /// Resolves the path against `root`, which must be the `Event` element itself. + /// + /// Returns `None` when any step fails to match, which is the normal case for an optional + /// field rather than an error: maps are written against a provider's superset of fields and + /// individual events legitimately omit some of them. + pub fn evaluate<'a>(&self, root: &'a EventNode) -> Option<&'a str> { + let mut steps = self.steps.iter(); + let first = steps.next()?; + if first.name != root.name || first.predicate.is_some() { + return None; + } + + let mut current = root; + for step in steps { + current = select(current, step)?; + } + + match &self.attribute { + Some(name) => current.attribute(name), + None => current.text.as_deref(), + } + } +} + +fn parse_step(segment: &str, expression: &str) -> Result { + let Some(open) = segment.find('[') else { + return Ok(Step { + name: segment.to_string(), + predicate: None, + }); + }; + + let malformed = || PathError::MalformedPredicate { + path: expression.to_string(), + step: segment.to_string(), + }; + + if !segment.ends_with(']') { + return Err(malformed()); + } + + let name = segment[..open].to_string(); + let inner = &segment[open + 1..segment.len() - 1]; + + let predicate = if let Some(rest) = inner.strip_prefix('@') { + let (attribute, raw_value) = rest.split_once('=').ok_or_else(malformed)?; + let value = raw_value + .strip_prefix('"') + .and_then(|v| v.strip_suffix('"')) + .or_else(|| { + raw_value + .strip_prefix('\'') + .and_then(|v| v.strip_suffix('\'')) + }) + .ok_or_else(malformed)?; + Predicate::AttributeEquals { + name: attribute.to_string(), + value: value.to_string(), + } + } else { + let index: usize = inner.parse().map_err(|_| malformed())?; + if index == 0 { + return Err(malformed()); + } + Predicate::Index(index) + }; + + Ok(Step { + name, + predicate: Some(predicate), + }) +} + +fn select<'a>(parent: &'a EventNode, step: &Step) -> Option<&'a EventNode> { + // Filtered inline rather than through EventNode::children_named so the step's lifetime stays + // independent of the node's; the iterator does not outlive this call. + let mut candidates = parent + .children + .iter() + .filter(|child| child.name == step.name); + match &step.predicate { + // Bare step takes the first match. Whether EvtxECmd instead joins repeated unnamed + // elements is not yet confirmed against the tool; see issue #539. + None => candidates.next(), + Some(Predicate::Index(index)) => candidates.nth(index - 1), + Some(Predicate::AttributeEquals { name, value }) => { + candidates.find(|child| child.attribute(name) == Some(value.as_str())) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn event() -> EventNode { + EventNode::new("Event") + .with_child( + EventNode::new("System") + .with_child(EventNode::new("Computer").with_text("RING0IVY24-01")) + .with_child( + EventNode::new("Correlation") + .with_attribute("ActivityID", "{2f8b0c1e-0000-0000-0000-000000000000}"), + ), + ) + .with_child( + EventNode::new("EventData") + .with_child( + EventNode::new("Data") + .with_attribute("Name", "SubjectUserName") + .with_text("adam"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "LogonType") + .with_text("10"), + ), + ) + .with_child( + EventNode::new("UserData").with_child( + EventNode::new("EventInfo") + .with_child(EventNode::new("Username").with_text("TEST\\adam")), + ), + ) + } + + fn eval(expression: &str) -> Option { + ValuePath::parse(expression) + .expect("path parses") + .evaluate(&event()) + .map(str::to_string) + } + + #[test] + fn resolves_named_event_data() { + assert_eq!( + eval(r#"/Event/EventData/Data[@Name="LogonType"]"#).as_deref(), + Some("10") + ); + } + + #[test] + fn resolves_single_quoted_predicate_value() { + assert_eq!( + eval("/Event/EventData/Data[@Name='SubjectUserName']").as_deref(), + Some("adam") + ); + } + + #[test] + fn bare_step_takes_the_first_match() { + assert_eq!(eval("/Event/EventData/Data").as_deref(), Some("adam")); + } + + #[test] + fn index_predicate_is_one_based() { + assert_eq!(eval("/Event/EventData/Data[1]").as_deref(), Some("adam")); + assert_eq!(eval("/Event/EventData/Data[2]").as_deref(), Some("10")); + assert_eq!(eval("/Event/EventData/Data[3]"), None); + } + + #[test] + fn resolves_nested_user_data() { + assert_eq!( + eval("/Event/UserData/EventInfo/Username").as_deref(), + Some("TEST\\adam") + ); + } + + #[test] + fn resolves_system_element_and_attribute() { + assert_eq!( + eval("/Event/System/Computer").as_deref(), + Some("RING0IVY24-01") + ); + assert_eq!( + eval("/Event/System/Correlation/@ActivityID").as_deref(), + Some("{2f8b0c1e-0000-0000-0000-000000000000}") + ); + } + + #[test] + fn missing_field_is_none_not_an_error() { + assert_eq!(eval(r#"/Event/EventData/Data[@Name="Absent"]"#), None); + assert_eq!(eval("/Event/EventData/Missing"), None); + } + + #[test] + fn container_without_text_resolves_to_none() { + assert_eq!(eval("/Event/EventData"), None); + } + + #[test] + fn rejects_relative_paths() { + assert!(matches!( + ValuePath::parse("Event/EventData"), + Err(PathError::NotAbsolute(_)) + )); + } + + #[test] + fn rejects_attribute_before_the_final_step() { + assert!(matches!( + ValuePath::parse("/Event/@Name/EventData"), + Err(PathError::MisplacedAttribute(_)) + )); + } + + #[test] + fn rejects_malformed_predicates() { + for expression in [ + "/Event/EventData/Data[@Name=\"unterminated", + "/Event/EventData/Data[@Name]", + "/Event/EventData/Data[abc]", + "/Event/EventData/Data[0]", + ] { + assert!( + matches!( + ValuePath::parse(expression), + Err(PathError::MalformedPredicate { .. }) + ), + "expected malformed predicate for {expression}" + ); + } + } + + #[test] + fn rejects_empty_paths() { + assert!(matches!(ValuePath::parse("/"), Err(PathError::Empty(_)))); + } + + #[test] + fn root_mismatch_resolves_to_none() { + let path = ValuePath::parse("/Other/EventData").expect("parses"); + assert_eq!(path.evaluate(&event()), None); + } +} diff --git a/crates/cmtraceopen-parser/src/lib.rs b/crates/cmtraceopen-parser/src/lib.rs index 2815a94fa..06834eff0 100644 --- a/crates/cmtraceopen-parser/src/lib.rs +++ b/crates/cmtraceopen-parser/src/lib.rs @@ -11,6 +11,7 @@ pub mod collector; pub mod dsregcmd; pub mod error_db; pub mod esp; +pub mod eventmap; pub mod intune; pub mod models; pub mod parser; diff --git a/crates/cmtraceopen-parser/tests/eventmap_corpus.rs b/crates/cmtraceopen-parser/tests/eventmap_corpus.rs new file mode 100644 index 000000000..fd2a1bc15 --- /dev/null +++ b/crates/cmtraceopen-parser/tests/eventmap_corpus.rs @@ -0,0 +1,209 @@ +//! The map engine driven by unmodified upstream EvtxECmd maps. +//! +//! These fixtures are real corpus (see `tests/fixtures/eventmap/README.md`), converted from YAML +//! to JSON without touching keys or values. The point is to prove the schema is implemented as +//! shipped rather than as imagined, so a regression here means the engine has drifted from the +//! community format. + +use cmtraceopen_parser::eventmap::{apply_map, EventMap, EventNode, MapProperty, MapRegistry}; + +const SHELL_CORE_9701: &str = include_str!("fixtures/eventmap/shell-core-9701.json"); +const SECURITY_4624: &str = include_str!("fixtures/eventmap/security-4624.json"); +const NTFS_146: &str = include_str!("fixtures/eventmap/ntfs-146-lookups.json"); + +fn load(raw: &str) -> EventMap { + serde_json::from_str(raw).expect("upstream map deserializes") +} + +/// Builds an `Event` whose `EventData` holds the given named `Data` elements. +fn event_with_named_data(fields: &[(&str, &str)]) -> EventNode { + let event_data = + fields + .iter() + .fold(EventNode::new("EventData"), |event_data, (name, value)| { + event_data.with_child( + EventNode::new("Data") + .with_attribute("Name", *name) + .with_text(*value), + ) + }); + EventNode::new("Event").with_child(event_data) +} + +#[test] +fn shell_core_9701_maps_unnamed_event_data() { + let map = load(SHELL_CORE_9701); + assert_eq!(map.event_id, 9701); + assert_eq!(map.channel, "Microsoft-Windows-Shell-Core/Operational"); + assert_eq!(map.author.as_deref(), Some("Troy Larson")); + + let event = EventNode::new("Event").with_child( + EventNode::new("EventData") + .with_child(EventNode::new("Data").with_text("RunOnceEx commands started")), + ); + + let mapped = apply_map(&map, &event); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("RunOnceEx commands started") + ); + assert!(mapped.invalid_paths.is_empty()); + assert!(mapped.values.iter().all(|value| value.is_complete())); +} + +#[test] +fn security_4624_fills_every_column_from_a_complete_event() { + let map = load(SECURITY_4624); + assert_eq!(map.maps.len(), 8); + + let event = event_with_named_data(&[ + ("SubjectDomainName", "TEST"), + ("SubjectUserName", "adam"), + ("IpAddress", "192.168.16.103"), + ("WorkstationName", "RING0IVY24-01"), + ("TargetDomainName", "TEST"), + ("TargetUserName", "svc-collector"), + ("LogonType", "10"), + ("TargetLogonId", "0x3e7"), + ("AuthenticationPackageName", "Negotiate"), + ("LogonProcessName", "Advapi"), + ("ProcessName", "C:\\Windows\\System32\\svchost.exe"), + ]); + + let mapped = apply_map(&map, &event); + + assert_eq!(mapped.value_for(&MapProperty::UserName), Some("TEST\\adam")); + assert_eq!( + mapped.value_for(&MapProperty::RemoteHost), + Some("RING0IVY24-01 (192.168.16.103)") + ); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("Target: TEST\\svc-collector") + ); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(2)), + Some("LogonType 10") + ); + assert_eq!( + mapped.value_for(&MapProperty::ExecutableInfo), + Some("C:\\Windows\\System32\\svchost.exe") + ); + assert!( + mapped.values.iter().all(|value| value.is_complete()), + "every column should resolve from a complete event" + ); +} + +#[test] +fn security_4624_reports_absent_fields_instead_of_blanking_them() { + let map = load(SECURITY_4624); + + // A real 4624 from a network logon carries no WorkstationName. + let event = event_with_named_data(&[ + ("SubjectDomainName", "TEST"), + ("SubjectUserName", "adam"), + ("IpAddress", "192.168.16.103"), + ]); + + let mapped = apply_map(&map, &event); + + assert_eq!(mapped.value_for(&MapProperty::UserName), Some("TEST\\adam")); + + let remote_host = mapped + .values + .iter() + .find(|value| value.property == MapProperty::RemoteHost) + .expect("RemoteHost column exists"); + assert_eq!(remote_host.unresolved, vec!["workstation".to_string()]); + assert!(!remote_host.is_complete()); + assert!( + remote_host.text.contains("192.168.16.103"), + "the field that did resolve is still shown: {}", + remote_host.text + ); + assert!( + remote_host.text.contains("%workstation%"), + "the missing field stays visibly unresolved rather than rendering as blank: {}", + remote_host.text + ); +} + +#[test] +fn ntfs_146_applies_its_lookup_table_and_default() { + let map = load(NTFS_146); + let lookup = map.lookup_for("BusType").expect("BusType lookup present"); + assert_eq!(lookup.default.as_deref(), Some("Unknown code")); + + let usb = apply_map( + &map, + &event_with_named_data(&[("VolumeName", "C:"), ("BusType", "7")]), + ); + assert!( + usb.values + .iter() + .any(|value| value.text.contains("USB") && !value.text.contains(": 7")), + "raw BusType 7 should render as USB: {:?}", + usb.values.iter().map(|v| &v.text).collect::>() + ); + + let unknown = apply_map( + &map, + &event_with_named_data(&[("VolumeName", "C:"), ("BusType", "255")]), + ); + assert!( + unknown + .values + .iter() + .any(|value| value.text.contains("Unknown code")), + "an out-of-table code should fall back to the lookup default" + ); +} + +#[test] +fn a_registry_resolves_each_fixture_by_its_own_identity() { + let mut registry = MapRegistry::new(); + for raw in [SHELL_CORE_9701, SECURITY_4624, NTFS_146] { + registry.insert(load(raw)); + } + assert_eq!(registry.len(), 3); + + let mapped = registry + .apply( + "Microsoft-Windows-Shell-Core/Operational", + "Microsoft-Windows-Shell-Core", + 9701, + &EventNode::new("Event").with_child( + EventNode::new("EventData").with_child(EventNode::new("Data").with_text("cmd.exe")), + ), + ) + .expect("registered map is found"); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("cmd.exe") + ); + + // Right channel and provider, wrong event: no map, and no guessing. + assert!(registry + .apply( + "Microsoft-Windows-Shell-Core/Operational", + "Microsoft-Windows-Shell-Core", + 9702, + &EventNode::new("Event") + ) + .is_none()); +} + +#[test] +fn every_fixture_parses_with_no_malformed_paths() { + for raw in [SHELL_CORE_9701, SECURITY_4624, NTFS_146] { + let map = load(raw); + let mapped = apply_map(&map, &EventNode::new("Event")); + assert!( + mapped.invalid_paths.is_empty(), + "upstream map {} has expressions this engine cannot parse: {:?}", + map.event_id, + mapped.invalid_paths + ); + } +} diff --git a/crates/cmtraceopen-parser/tests/fixtures/eventmap/README.md b/crates/cmtraceopen-parser/tests/fixtures/eventmap/README.md new file mode 100644 index 000000000..bbde5fecb --- /dev/null +++ b/crates/cmtraceopen-parser/tests/fixtures/eventmap/README.md @@ -0,0 +1,14 @@ +# EvtxECmd map fixtures + +These are unmodified EvtxECmd maps from [EricZimmerman/evtx](https://github.com/EricZimmerman/evtx) +(MIT licensed), converted from YAML to JSON. Only the serialization format changed; keys, values, +and structure are byte-faithful to the upstream corpus. + +They are vendored so the map engine is tested against the real schema rather than invented +examples, per the repository rule that fixtures must anchor to real corpus. + +| Fixture | Upstream file | Why it is here | +|---|---|---| +| `shell-core-9701.json` | `Microsoft-Windows-Shell-Core-Operational_Microsoft-Windows-Shell-Core_9701.map` | Simplest shape: one entry, bare `/Event/EventData/Data`. RunOnceEx during Autopilot OOBE. | +| `security-4624.json` | `Security_Microsoft-Windows-Security-Auditing_4624.map` | Multiple bindings per entry, multi-placeholder templates, `UserName` and `RemoteHost` targets. | +| `ntfs-146-lookups.json` | `Microsoft-Windows-Ntfs-Operational_Microsoft-Windows-Ntfs_146.map` | Exercises `Lookups` with a `Default` fallback. | diff --git a/crates/cmtraceopen-parser/tests/fixtures/eventmap/ntfs-146-lookups.json b/crates/cmtraceopen-parser/tests/fixtures/eventmap/ntfs-146-lookups.json new file mode 100644 index 000000000..8db122b40 --- /dev/null +++ b/crates/cmtraceopen-parser/tests/fixtures/eventmap/ntfs-146-lookups.json @@ -0,0 +1,96 @@ +{ + "Author": "Hyun Yi @hyuunnn", + "Description": "IO Connection", + "EventId": 146, + "Channel": "Microsoft-Windows-Ntfs/Operational", + "Provider": "Microsoft-Windows-Ntfs", + "Maps": [ + { + "Property": "PayloadData1", + "PropertyValue": "VolumeName: %VolumeName%", + "Values": [ + { + "Name": "VolumeName", + "Value": "/Event/EventData/Data[@Name=\"VolumeName\"]" + } + ] + }, + { + "Property": "PayloadData2", + "PropertyValue": "IsBootVolume: %IsBootVolume%", + "Values": [ + { + "Name": "IsBootVolume", + "Value": "/Event/EventData/Data[@Name=\"IsBootVolume\"]" + } + ] + }, + { + "Property": "PayloadData3", + "PropertyValue": "VendorId: %VendorId%", + "Values": [ + { + "Name": "VendorId", + "Value": "/Event/EventData/Data[@Name=\"VendorId\"]" + } + ] + }, + { + "Property": "PayloadData4", + "PropertyValue": "ProductId: %ProductId%", + "Values": [ + { + "Name": "ProductId", + "Value": "/Event/EventData/Data[@Name=\"ProductId\"]" + } + ] + }, + { + "Property": "PayloadData5", + "PropertyValue": "DeviceSerialNumber: %DeviceSerialNumber%", + "Values": [ + { + "Name": "DeviceSerialNumber", + "Value": "/Event/EventData/Data[@Name=\"DeviceSerialNumber\"]" + } + ] + }, + { + "Property": "PayloadData6", + "PropertyValue": "BusType: %BusType%", + "Values": [ + { + "Name": "BusType", + "Value": "/Event/EventData/Data[@Name=\"BusType\"]" + } + ] + } + ], + "Lookups": [ + { + "Name": "BusType", + "Default": "Unknown code", + "Values": { + "0": "The bus type is unknown.", + "1": "SCSI", + "2": "ATAPI", + "3": "ATA", + "4": "IEEE 1394", + "5": "SSA", + "6": "Fibre Channel", + "7": "USB", + "8": "RAID", + "9": "iSCSI", + "10": "Serial Attached SCSI (SAS)", + "11": "Serial ATA (SATA)", + "12": "Secure Digital (SD)", + "13": "Multimedia Card (MMC)", + "14": "This value is reserved for system use.", + "15": "File-Backed Virtual", + "16": "Storage Spaces", + "17": "NVMe", + "18": "This value is reserved for system use." + } + } + ] +} diff --git a/crates/cmtraceopen-parser/tests/fixtures/eventmap/security-4624.json b/crates/cmtraceopen-parser/tests/fixtures/eventmap/security-4624.json new file mode 100644 index 000000000..af92122a3 --- /dev/null +++ b/crates/cmtraceopen-parser/tests/fixtures/eventmap/security-4624.json @@ -0,0 +1,101 @@ +{ + "Author": "Eric Zimmerman saericzimmerman@gmail.com", + "Description": "Successful logon", + "EventId": 4624, + "Channel": "Security", + "Provider": "Microsoft-Windows-Security-Auditing", + "Maps": [ + { + "Property": "UserName", + "PropertyValue": "%domain%\\%user%", + "Values": [ + { + "Name": "domain", + "Value": "/Event/EventData/Data[@Name=\"SubjectDomainName\"]" + }, + { + "Name": "user", + "Value": "/Event/EventData/Data[@Name=\"SubjectUserName\"]" + } + ] + }, + { + "Property": "RemoteHost", + "PropertyValue": "%workstation% (%ipAddress%)", + "Values": [ + { + "Name": "ipAddress", + "Value": "/Event/EventData/Data[@Name=\"IpAddress\"]" + }, + { + "Name": "workstation", + "Value": "/Event/EventData/Data[@Name=\"WorkstationName\"]" + } + ] + }, + { + "Property": "PayloadData1", + "PropertyValue": "Target: %TargetDomainName%\\%TargetUserName%", + "Values": [ + { + "Name": "TargetDomainName", + "Value": "/Event/EventData/Data[@Name=\"TargetDomainName\"]" + }, + { + "Name": "TargetUserName", + "Value": "/Event/EventData/Data[@Name=\"TargetUserName\"]" + } + ] + }, + { + "Property": "PayloadData2", + "PropertyValue": "LogonType %LogonType%", + "Values": [ + { + "Name": "LogonType", + "Value": "/Event/EventData/Data[@Name=\"LogonType\"]" + } + ] + }, + { + "Property": "PayloadData3", + "PropertyValue": "LogonId: %TargetLogonId%", + "Values": [ + { + "Name": "TargetLogonId", + "Value": "/Event/EventData/Data[@Name=\"TargetLogonId\"]" + } + ] + }, + { + "Property": "PayloadData4", + "PropertyValue": "AuthenticationPackageName: %AuthenticationPackageName%", + "Values": [ + { + "Name": "AuthenticationPackageName", + "Value": "/Event/EventData/Data[@Name=\"AuthenticationPackageName\"]" + } + ] + }, + { + "Property": "PayloadData5", + "PropertyValue": "LogonProcessName: %LogonProcessName%", + "Values": [ + { + "Name": "LogonProcessName", + "Value": "/Event/EventData/Data[@Name=\"LogonProcessName\"]" + } + ] + }, + { + "Property": "ExecutableInfo", + "PropertyValue": "%ProcessName%", + "Values": [ + { + "Name": "ProcessName", + "Value": "/Event/EventData/Data[@Name=\"ProcessName\"]" + } + ] + } + ] +} diff --git a/crates/cmtraceopen-parser/tests/fixtures/eventmap/shell-core-9701.json b/crates/cmtraceopen-parser/tests/fixtures/eventmap/shell-core-9701.json new file mode 100644 index 000000000..ec895ed90 --- /dev/null +++ b/crates/cmtraceopen-parser/tests/fixtures/eventmap/shell-core-9701.json @@ -0,0 +1,19 @@ +{ + "Author": "Troy Larson", + "Description": "RunOnceEx commands started", + "EventId": 9701, + "Channel": "Microsoft-Windows-Shell-Core/Operational", + "Provider": "Microsoft-Windows-Shell-Core", + "Maps": [ + { + "Property": "PayloadData1", + "PropertyValue": "%PayloadData1%", + "Values": [ + { + "Name": "PayloadData1", + "Value": "/Event/EventData/Data" + } + ] + } + ] +} From 23dca7b427758a1e438629fc16d09ec074c856a0 Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 9 Aug 2026 13:01:45 -0400 Subject: [PATCH 02/85] fix(parser): join repeated elements, matching verified EvtxECmd behaviour A bare step such as /Event/EventData/Data took the first matching element. That was a guess, flagged as an open question in the previous commit, and it was wrong. Verified against EvtxECmd 1.5.2 on a Windows 11 lab host rather than reasoned about. A probe map binding /Event/EventData/Data was run over a real ESENT event ID 326 record carrying nine unnamed children, with a maps directory containing only that map. The emitted PayloadData1 was 1,712 characters longer than the first element alone, and the bytes between the first and second element were 44 and 32, so repeated elements are joined with ", ". Only the final step can select a repeated set, because joining containers has no meaning, and an attribute selector still reads a single element since joining attribute values across siblings would be meaningless too. A single match keeps its previous behaviour exactly, so paths that resolve one element, which is every named-Data expression and 78% of the corpus, are unaffected. evaluate now returns Cow so the common single-match case still borrows and only a genuine join allocates. Gates: 2,230 tests pass, clippy -D warnings clean, rustfmt clean, wasm32 target still builds. Refs #539. Co-Authored-By: Claude Opus 5 --- .../cmtraceopen-parser/src/eventmap/apply.rs | 2 +- .../cmtraceopen-parser/src/eventmap/path.rs | 99 ++++++++++++++++--- 2 files changed, 86 insertions(+), 15 deletions(-) diff --git a/crates/cmtraceopen-parser/src/eventmap/apply.rs b/crates/cmtraceopen-parser/src/eventmap/apply.rs index 6c2aa2c0c..d9f1d2183 100644 --- a/crates/cmtraceopen-parser/src/eventmap/apply.rs +++ b/crates/cmtraceopen-parser/src/eventmap/apply.rs @@ -79,7 +79,7 @@ fn apply_entry( } let resolved = match ValuePath::parse(&binding.value) { - Ok(path) => path.evaluate(event).map(str::to_string), + Ok(path) => path.evaluate(event).map(|value| value.into_owned()), Err(_) => { let defect = (binding.name.clone(), binding.value.clone()); if !invalid_paths.contains(&defect) { diff --git a/crates/cmtraceopen-parser/src/eventmap/path.rs b/crates/cmtraceopen-parser/src/eventmap/path.rs index b0c300d15..4aac2b1dd 100644 --- a/crates/cmtraceopen-parser/src/eventmap/path.rs +++ b/crates/cmtraceopen-parser/src/eventmap/path.rs @@ -16,6 +16,8 @@ //! 1-based index predicate, optionally ending in an attribute selector. A general XPath engine //! would be far more machinery than the grammar justifies. +use std::borrow::Cow; + use thiserror::Error; use super::node::EventNode; @@ -97,25 +99,74 @@ impl ValuePath { /// Returns `None` when any step fails to match, which is the normal case for an optional /// field rather than an error: maps are written against a provider's superset of fields and /// individual events legitimately omit some of them. - pub fn evaluate<'a>(&self, root: &'a EventNode) -> Option<&'a str> { - let mut steps = self.steps.iter(); - let first = steps.next()?; + pub fn evaluate<'a>(&self, root: &'a EventNode) -> Option> { + let first = self.steps.first()?; if first.name != root.name || first.predicate.is_some() { return None; } + // Every step but the last narrows to a single container. Only the final step can select + // a repeated set, because joining containers has no meaning. let mut current = root; - for step in steps { - current = select(current, step)?; + let mut remaining = &self.steps[1..]; + while remaining.len() > 1 { + current = select_one(current, &remaining[0])?; + remaining = &remaining[1..]; } - match &self.attribute { - Some(name) => current.attribute(name), - None => current.text.as_deref(), + match remaining.first() { + None => read(current, self.attribute.as_deref()), + Some(step) => read_final(current, step, self.attribute.as_deref()), } } } +/// Separator EvtxECmd uses when a bare step matches repeated elements. +/// +/// Verified against EvtxECmd itself rather than assumed. A probe map binding +/// `/Event/EventData/Data` was run over a real `ESENT` event ID 326 record carrying nine unnamed +/// `` children: the emitted `PayloadData1` was 1,712 characters longer than the first +/// element alone, and the bytes between elements were 44, 32. +const REPEATED_ELEMENT_SEPARATOR: &str = ", "; + +fn read<'a>(node: &'a EventNode, attribute: Option<&str>) -> Option> { + match attribute { + Some(name) => node.attribute(name).map(Cow::Borrowed), + None => node.text.as_deref().map(Cow::Borrowed), + } +} + +fn read_final<'a>( + parent: &'a EventNode, + step: &Step, + attribute: Option<&str>, +) -> Option> { + if step.predicate.is_some() { + return read(select_one(parent, step)?, attribute); + } + + let matches: Vec<&EventNode> = parent + .children + .iter() + .filter(|child| child.name == step.name) + .collect(); + + match matches.as_slice() { + [] => None, + [only] => read(only, attribute), + // An attribute selector still reads one element; only text content is joined. + many => match attribute { + Some(name) => many[0].attribute(name).map(Cow::Borrowed), + None => Some(Cow::Owned( + many.iter() + .map(|node| node.text.as_deref().unwrap_or_default()) + .collect::>() + .join(REPEATED_ELEMENT_SEPARATOR), + )), + }, + } +} + fn parse_step(segment: &str, expression: &str) -> Result { let Some(open) = segment.find('[') else { return Ok(Step { @@ -165,7 +216,7 @@ fn parse_step(segment: &str, expression: &str) -> Result { }) } -fn select<'a>(parent: &'a EventNode, step: &Step) -> Option<&'a EventNode> { +fn select_one<'a>(parent: &'a EventNode, step: &Step) -> Option<&'a EventNode> { // Filtered inline rather than through EventNode::children_named so the step's lifetime stays // independent of the node's; the iterator does not outlive this call. let mut candidates = parent @@ -173,8 +224,6 @@ fn select<'a>(parent: &'a EventNode, step: &Step) -> Option<&'a EventNode> { .iter() .filter(|child| child.name == step.name); match &step.predicate { - // Bare step takes the first match. Whether EvtxECmd instead joins repeated unnamed - // elements is not yet confirmed against the tool; see issue #539. None => candidates.next(), Some(Predicate::Index(index)) => candidates.nth(index - 1), Some(Predicate::AttributeEquals { name, value }) => { @@ -222,7 +271,7 @@ mod tests { ValuePath::parse(expression) .expect("path parses") .evaluate(&event()) - .map(str::to_string) + .map(|value| value.into_owned()) } #[test] @@ -242,8 +291,30 @@ mod tests { } #[test] - fn bare_step_takes_the_first_match() { - assert_eq!(eval("/Event/EventData/Data").as_deref(), Some("adam")); + fn bare_step_joins_repeated_elements_as_evtxecmd_does() { + // Verified against EvtxECmd on a real ESENT 326 record; see REPEATED_ELEMENT_SEPARATOR. + assert_eq!(eval("/Event/EventData/Data").as_deref(), Some("adam, 10")); + } + + #[test] + fn bare_step_with_a_single_match_returns_that_element_untouched() { + let single = EventNode::new("Event").with_child( + EventNode::new("EventData").with_child(EventNode::new("Data").with_text("RunOnceEx")), + ); + let path = ValuePath::parse("/Event/EventData/Data").expect("parses"); + assert_eq!(path.evaluate(&single).as_deref(), Some("RunOnceEx")); + } + + #[test] + fn joining_preserves_position_for_an_element_with_no_text() { + let gapped = EventNode::new("Event").with_child( + EventNode::new("EventData") + .with_child(EventNode::new("Data").with_text("first")) + .with_child(EventNode::new("Data")) + .with_child(EventNode::new("Data").with_text("third")), + ); + let path = ValuePath::parse("/Event/EventData/Data").expect("parses"); + assert_eq!(path.evaluate(&gapped).as_deref(), Some("first, , third")); } #[test] From 4fe3ab700bbbba28225c0b7324710228bea0f2a8 Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 9 Aug 2026 13:14:04 -0400 Subject: [PATCH 03/85] feat(event-log): load EvtxECmd .map files from disk Host-side adapter for the map engine. The schema and engine stay in cmtraceopen-parser, which is pure and wasm32-compatible and therefore carries no YAML dependency; serde_norway is added to src-tauri only, where I/O already lives. The parser crate is unchanged by this commit and still builds for wasm32-unknown-unknown. Two behaviours were verified against EvtxECmd 1.5.2 on a Windows 11 host rather than inferred from its documentation: - First loaded wins. Two maps claiming the same identity were placed in one directory; EvtxECmd rejected the second with "An item with the same key has already been added. Key: 326-APPLICATION-ESENT", reported "Maps loaded: 1", and kept the 1_-prefixed file. That is the opposite of MapRegistry::insert, which is last-wins, so the loader checks ownership before inserting rather than relying on registry semantics. - Identity is case-insensitive, and that same key shows channel and provider uppercased, matching how MapRegistry compares them. Upstream .map files are UTF-8 with a byte order mark, which YAML parsers reject as an unexpected character, so the BOM is stripped before deserializing. Failures and supersessions are reported rather than dropped. A map that did not load means events of that type render unmapped, which is a coverage gap an operator needs to see, not a silent omission. Gates: 1,095 src-tauri tests pass, clippy -D warnings clean, rustfmt clean, tsc clean, cargo audit exits 0 for the new dependency, and the parser crate still builds for wasm32. Refs #539. Co-Authored-By: Claude Opus 5 --- Cargo.lock | 26 +++ src-tauri/Cargo.toml | 1 + src-tauri/src/event_log/maps.rs | 300 ++++++++++++++++++++++++++++++++ src-tauri/src/event_log/mod.rs | 1 + 4 files changed, 328 insertions(+) create mode 100644 src-tauri/src/event_log/maps.rs diff --git a/Cargo.lock b/Cargo.lock index ab7d62a2a..48bb7ca8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -599,6 +599,7 @@ dependencies = [ "regex", "serde", "serde_json", + "serde_norway", "sha2 0.11.0", "tauri", "tauri-build", @@ -3977,6 +3978,12 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "same-file" version = "1.0.6" @@ -4191,6 +4198,19 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_norway" +version = "0.9.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e408f29489b5fd500fab51ff1484fc859bb655f32c671f307dcd733b72e8168c" +dependencies = [ + "indexmap 2.11.4", + "itoa", + "ryu", + "serde", + "unsafe-libyaml-norway", +] + [[package]] name = "serde_repr" version = "0.1.20" @@ -5531,6 +5551,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "unsafe-libyaml-norway" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39abd59bf32521c7f2301b52d05a6a2c975b6003521cbd0c6dc1582f0a22104" + [[package]] name = "untrusted" version = "0.9.0" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index fafd291c2..cfbede6b5 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -75,6 +75,7 @@ cab = { version = "=0.6.0", optional = true } # using only std::time -- so the whole graph (cab, cookie, plist, simplelog, # tauri-plugin-log, tauri-plugin-updater) is held at or above the patched release. time = { version = "0.3.47", default-features = false, features = ["parsing"] } +serde_norway = "0.9.42" [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/src-tauri/src/event_log/maps.rs b/src-tauri/src/event_log/maps.rs new file mode 100644 index 000000000..ffe490cb0 --- /dev/null +++ b/src-tauri/src/event_log/maps.rs @@ -0,0 +1,300 @@ +//! Loading EvtxECmd `.map` files from disk. +//! +//! The schema and the engine live in `cmtraceopen-parser`, which stays pure and +//! wasm32-compatible and therefore has no YAML dependency. This module is the host-side adapter: +//! it reads `.map` files, deserializes the YAML, and builds a +//! [`MapRegistry`](cmtraceopen_parser::eventmap::MapRegistry). +//! +//! Two behaviours were verified against EvtxECmd 1.5.2 on a Windows 11 host rather than inferred +//! from its documentation: +//! +//! - **First loaded wins.** Files load in alphabetical order and a later file with the same +//! identity is rejected, which is what makes the documented `1_` prefix override work. EvtxECmd +//! reports this as `An item with the same key has already been added. Key: 326-APPLICATION-ESENT` +//! and continues with the first map. +//! - **Identity is case-insensitive.** That same key is uppercased channel and provider, matching +//! how `MapRegistry` compares them. + +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use cmtraceopen_parser::eventmap::{EventMap, MapRegistry}; +use serde::Serialize; + +/// Why a `.map` file could not be used. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct MapLoadFailure { + /// The file that failed. + pub path: String, + /// The reason, suitable for showing to an operator. + pub reason: String, +} + +/// A map that parsed but lost to an earlier file claiming the same identity. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct SupersededMap { + /// The file that was skipped. + pub path: String, + /// The file that already owned this identity. + pub superseded_by: String, + /// `channel/provider/eventId` of the contested identity. + pub identity: String, +} + +/// The result of loading a directory of maps. +/// +/// Failures and supersessions are reported rather than silently dropped: a map that did not load +/// means events of that type render unmapped, which is a coverage gap an operator needs to see. +#[derive(Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MapLoadOutcome { + /// Files that loaded and won their identity. + pub loaded: Vec, + /// Files skipped because an earlier file already owned the identity. + pub superseded: Vec, + /// Files that could not be read or parsed. + pub failures: Vec, +} + +impl MapLoadOutcome { + /// Number of maps actually registered. + pub fn loaded_count(&self) -> usize { + self.loaded.len() + } + + /// True when nothing failed and nothing was skipped. + pub fn is_clean(&self) -> bool { + self.failures.is_empty() && self.superseded.is_empty() + } +} + +/// Parses one `.map` file's contents. +/// +/// Upstream maps are UTF-8 with a byte order mark, which YAML parsers reject as an unexpected +/// character, so the BOM is stripped first. +pub fn parse_map(contents: &str) -> Result { + let without_bom = contents.strip_prefix('\u{feff}').unwrap_or(contents); + serde_norway::from_str::(without_bom).map_err(|error| error.to_string()) +} + +fn identity_of(map: &EventMap) -> String { + format!("{}/{}/{}", map.channel, map.provider, map.event_id) +} + +/// Loads every `.map` file directly inside `directory`. +/// +/// Files are processed in case-insensitive alphabetical order so a `1_`-prefixed copy wins, which +/// is how upstream lets a local customization survive a map-corpus update. +pub fn load_maps_from_dir(directory: &Path) -> Result<(MapRegistry, MapLoadOutcome), String> { + let entries = fs::read_dir(directory) + .map_err(|error| format!("cannot read map directory {}: {error}", directory.display()))?; + + let mut files: Vec = entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| { + path.is_file() + && path + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("map")) + }) + .collect(); + files.sort_by_key(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default() + .to_ascii_lowercase() + }); + + let mut registry = MapRegistry::new(); + let mut outcome = MapLoadOutcome::default(); + // identity (lowercased, matching MapRegistry's comparison) -> the file that claimed it + let mut owners: HashMap = HashMap::new(); + + for path in files { + let display = path.display().to_string(); + let contents = match fs::read_to_string(&path) { + Ok(contents) => contents, + Err(error) => { + outcome.failures.push(MapLoadFailure { + path: display, + reason: format!("cannot read file: {error}"), + }); + continue; + } + }; + + let map = match parse_map(&contents) { + Ok(map) => map, + Err(reason) => { + outcome.failures.push(MapLoadFailure { + path: display, + reason, + }); + continue; + } + }; + + // First loaded wins, matching EvtxECmd. Checking before inserting is deliberate: + // MapRegistry::insert is last-wins, which is the opposite of what upstream does. + let identity = identity_of(&map); + let owner_key = identity.to_ascii_lowercase(); + if let Some(owner) = owners.get(&owner_key) { + outcome.superseded.push(SupersededMap { + path: display, + superseded_by: owner.clone(), + identity, + }); + continue; + } + + owners.insert(owner_key, display.clone()); + registry.insert(map); + outcome.loaded.push(display); + } + + Ok((registry, outcome)) +} + +#[cfg(test)] +mod tests { + use super::*; + use cmtraceopen_parser::eventmap::{apply_map, EventNode, MapProperty}; + use std::fs; + + /// A real upstream map, byte for byte, including the leading comment style and quoting. + const SHELL_CORE_9701: &str = r#"Author: Troy Larson +Description: RunOnceEx commands started +EventId: 9701 +Channel: Microsoft-Windows-Shell-Core/Operational +Provider: Microsoft-Windows-Shell-Core +Maps: + - + Property: PayloadData1 + PropertyValue: "%PayloadData1%" + Values: + - + Name: PayloadData1 + Value: "/Event/EventData/Data" + +# Documentation: +# https://www.geoffchappell.com/notes/windows/shell/events/core.htm +"#; + + fn temp_dir(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("cmtraceopen-maps-{name}")); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("temp dir"); + dir + } + + fn write(dir: &Path, name: &str, contents: &str) { + fs::write(dir.join(name), contents).expect("write map"); + } + + #[test] + fn parses_a_real_upstream_map_including_trailing_comments() { + let map = parse_map(SHELL_CORE_9701).expect("parses"); + assert_eq!(map.event_id, 9701); + assert_eq!(map.provider, "Microsoft-Windows-Shell-Core"); + assert_eq!(map.author.as_deref(), Some("Troy Larson")); + assert_eq!(map.maps.len(), 1); + assert_eq!(map.maps[0].property, MapProperty::PayloadData(1)); + } + + #[test] + fn strips_the_utf8_byte_order_mark_upstream_files_carry() { + let with_bom = format!("\u{feff}{SHELL_CORE_9701}"); + let map = parse_map(&with_bom).expect("BOM-prefixed map parses"); + assert_eq!(map.event_id, 9701); + } + + #[test] + fn reports_a_malformed_map_as_a_failure_rather_than_aborting_the_load() { + let dir = temp_dir("malformed"); + write(&dir, "good.map", SHELL_CORE_9701); + write(&dir, "bad.map", "EventId: [this is not a scalar\nChannel:"); + + let (registry, outcome) = load_maps_from_dir(&dir).expect("directory reads"); + + assert_eq!(registry.len(), 1); + assert_eq!(outcome.loaded_count(), 1); + assert_eq!(outcome.failures.len(), 1); + assert!(outcome.failures[0].path.ends_with("bad.map")); + assert!(!outcome.is_clean()); + } + + #[test] + fn first_loaded_wins_so_a_numeric_prefix_overrides() { + let dir = temp_dir("override"); + let original = SHELL_CORE_9701.replace("%PayloadData1%", "ORIGINAL %PayloadData1%"); + let custom = SHELL_CORE_9701.replace("%PayloadData1%", "CUSTOM %PayloadData1%"); + write(&dir, "Shell-Core_9701.map", &original); + write(&dir, "1_Shell-Core_9701.map", &custom); + + let (registry, outcome) = load_maps_from_dir(&dir).expect("directory reads"); + + assert_eq!(registry.len(), 1); + assert_eq!(outcome.superseded.len(), 1); + assert!(outcome.superseded[0].path.ends_with("Shell-Core_9701.map")); + assert!(outcome.superseded[0].superseded_by.contains("1_")); + + let event = EventNode::new("Event").with_child( + EventNode::new("EventData").with_child(EventNode::new("Data").with_text("cmd.exe")), + ); + let map = registry + .find( + "Microsoft-Windows-Shell-Core/Operational", + "Microsoft-Windows-Shell-Core", + 9701, + ) + .expect("map registered"); + assert_eq!( + apply_map(map, &event).value_for(&MapProperty::PayloadData(1)), + Some("CUSTOM cmd.exe"), + "the 1_-prefixed file loads first and must win" + ); + } + + #[test] + fn ignores_files_that_are_not_maps() { + let dir = temp_dir("filter"); + write(&dir, "real.map", SHELL_CORE_9701); + write(&dir, "notes.txt", "ignore me"); + write(&dir, "README.md", "ignore me too"); + + let (registry, outcome) = load_maps_from_dir(&dir).expect("directory reads"); + + assert_eq!(registry.len(), 1); + assert!(outcome.is_clean()); + } + + #[test] + fn accepts_an_uppercase_extension() { + let dir = temp_dir("uppercase"); + write(&dir, "real.MAP", SHELL_CORE_9701); + + let (registry, _) = load_maps_from_dir(&dir).expect("directory reads"); + assert_eq!(registry.len(), 1); + } + + #[test] + fn an_empty_directory_loads_cleanly_rather_than_erroring() { + let dir = temp_dir("empty"); + let (registry, outcome) = load_maps_from_dir(&dir).expect("directory reads"); + assert!(registry.is_empty()); + assert!(outcome.is_clean()); + assert_eq!(outcome.loaded_count(), 0); + } + + #[test] + fn a_missing_directory_is_an_error_not_an_empty_result() { + let missing = std::env::temp_dir().join("cmtraceopen-maps-does-not-exist"); + let _ = fs::remove_dir_all(&missing); + assert!(load_maps_from_dir(&missing).is_err()); + } +} diff --git a/src-tauri/src/event_log/mod.rs b/src-tauri/src/event_log/mod.rs index 3962e3dcb..1c286599b 100644 --- a/src-tauri/src/event_log/mod.rs +++ b/src-tauri/src/event_log/mod.rs @@ -1,4 +1,5 @@ pub mod commands; +pub mod maps; pub mod models; pub mod parser; From 6f819fded42eeb52ab178e2d70a7a34eac5cb676 Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 9 Aug 2026 14:05:09 -0400 Subject: [PATCH 04/85] fix(parser): render map templates in a single pass Addresses three review findings on the map engine, two of them correctness defects with concrete repro cases now covered by tests. Substituted values were re-scanned as templates. apply_entry tested text.contains against the partially rendered output and then used str::replace, so a resolved value that itself contained %Name% became a substitution target for a later binding. Event field content is untrusted: with security-4624 and SubjectDomainName set to the literal "%user%", UserName rendered as "adam\adam" instead of "%user%\adam". Bindings are now resolved against the original template and the result is rendered left to right in one pass, so substituted text is appended and never revisited. The placeholder scanner lost a name after rejecting a candidate. On finding a non-placeholder between two percent signs it advanced past the closing one, consuming the opening delimiter of the next real placeholder. "50% off %Cost%" therefore reported nothing unresolved while the text still read %Cost%, contradicting the documented contract of MappedValue::unresolved. The renderer now resumes immediately after the opening percent so the closing one stays available. EventNode::children_named tied the name's lifetime to the node borrow, which forced path::select_one to reimplement the filter inline. The name now carries its own lifetime and select_one uses the helper. Gates: clippy -D warnings clean, rustfmt clean, wasm32 target builds, and the crate's tests pass including four new cases covering percent handling. Refs #539. Co-Authored-By: Claude Opus 5 --- .../cmtraceopen-parser/src/eventmap/apply.rs | 169 +++++++++++++++--- .../cmtraceopen-parser/src/eventmap/node.rs | 8 +- .../cmtraceopen-parser/src/eventmap/path.rs | 7 +- 3 files changed, 154 insertions(+), 30 deletions(-) diff --git a/crates/cmtraceopen-parser/src/eventmap/apply.rs b/crates/cmtraceopen-parser/src/eventmap/apply.rs index d9f1d2183..26c691ced 100644 --- a/crates/cmtraceopen-parser/src/eventmap/apply.rs +++ b/crates/cmtraceopen-parser/src/eventmap/apply.rs @@ -4,6 +4,8 @@ //! fields, so an individual event legitimately omits some of them. A missing field is a coverage //! state, reported on [`MappedValue::unresolved`], never an error and never silently blank. +use std::collections::BTreeMap; + use super::model::{EventMap, MapEntry, MapProperty}; use super::node::EventNode; use super::path::ValuePath; @@ -69,16 +71,16 @@ fn apply_entry( event: &EventNode, invalid_paths: &mut Vec<(String, String)>, ) -> MappedValue { - let mut text = entry.property_value.clone(); - let mut unresolved = Vec::new(); + let template = entry.property_value.as_str(); + let mut resolved: BTreeMap<&str, String> = BTreeMap::new(); for binding in &entry.values { - let placeholder = format!("%{}%", binding.name); - if !text.contains(&placeholder) { + // Checked against the original template, never against partially rendered output. + if !template.contains(&format!("%{}%", binding.name)) { continue; } - let resolved = match ValuePath::parse(&binding.value) { + let value = match ValuePath::parse(&binding.value) { Ok(path) => path.evaluate(event).map(|value| value.into_owned()), Err(_) => { let defect = (binding.name.clone(), binding.value.clone()); @@ -89,24 +91,19 @@ fn apply_entry( } }; - let translated = resolved.and_then(|raw| match map.lookup_for(&binding.name) { + let translated = value.and_then(|raw| match map.lookup_for(&binding.name) { Some(lookup) => lookup.translate(&raw), None => Some(raw), }); - match translated { - Some(value) => text = text.replace(&placeholder, &value), - None => unresolved.push(binding.name.clone()), + if let Some(translated) = translated { + resolved.insert(binding.name.as_str(), translated); } } - // A template may reference a variable the map never binds. That is still an unresolved - // placeholder from the reader's point of view, so report it the same way. - for name in remaining_placeholders(&text) { - if !unresolved.contains(&name) { - unresolved.push(name); - } - } + // A binding that failed to resolve is simply absent from the map, so the renderer reports it + // alongside placeholders the map never bound at all. Both are the same thing to a reader. + let (text, unresolved) = render(template, &resolved); MappedValue { property: entry.property.clone(), @@ -115,19 +112,53 @@ fn apply_entry( } } -fn remaining_placeholders(text: &str) -> Vec { - let mut names = Vec::new(); - let mut rest = text; +/// Renders `template` in one left-to-right pass, returning the text and any unfilled placeholders. +/// +/// Single-pass matters for correctness, not just speed. Event field content is untrusted: a field +/// whose value is literally `%user%` must not become a substitution target for a later binding, +/// which is exactly what repeated `str::replace` over accumulating output would do. +fn render(template: &str, values: &BTreeMap<&str, String>) -> (String, Vec) { + let mut out = String::with_capacity(template.len()); + let mut unresolved: Vec = Vec::new(); + let mut rest = template; + while let Some(start) = rest.find('%') { + out.push_str(&rest[..start]); let after = &rest[start + 1..]; - let Some(end) = after.find('%') else { break }; + + let Some(end) = after.find('%') else { + // Unpaired '%': the remainder is literal text. + out.push_str(&rest[start..]); + return (out, unresolved); + }; + let name = &after[..end]; - if !name.is_empty() && !name.contains(char::is_whitespace) { - names.push(name.to_string()); + if name.is_empty() || name.contains(char::is_whitespace) { + // Not a placeholder. Emit the opening '%' and resume at the character after it, so the + // closing '%' stays available as the opening delimiter of a real placeholder that + // follows, as in "50% off %Cost%". + out.push('%'); + rest = after; + continue; + } + + match values.get(name) { + // The substituted value is appended to `out` and never revisited. + Some(value) => out.push_str(value), + None => { + out.push('%'); + out.push_str(name); + out.push('%'); + if !unresolved.iter().any(|existing| existing == name) { + unresolved.push(name.to_string()); + } + } } rest = &after[end + 1..]; } - names + + out.push_str(rest); + (out, unresolved) } #[cfg(test)] @@ -313,6 +344,98 @@ mod tests { assert_eq!(mapped.values[0].unresolved, vec!["broken".to_string()]); } + #[test] + fn a_field_value_containing_a_placeholder_is_not_re_substituted() { + // Event content is untrusted. If rendering re-scanned its own output, a field whose text + // happens to be "%user%" would be replaced by a later binding. + let event = EventNode::new("Event").with_child( + EventNode::new("EventData") + .with_child( + EventNode::new("Data") + .with_attribute("Name", "SubjectDomainName") + .with_text("%user%"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "SubjectUserName") + .with_text("adam"), + ), + ); + let map = map_with( + vec![MapEntry { + property: MapProperty::UserName, + property_value: "%domain%\\%user%".to_string(), + values: vec![ + binding("domain", "SubjectDomainName"), + binding("user", "SubjectUserName"), + ], + }], + vec![], + ); + + let mapped = apply_map(&map, &event); + assert_eq!( + mapped.value_for(&MapProperty::UserName), + Some("%user%\\adam") + ); + } + + #[test] + fn a_literal_percent_does_not_hide_the_placeholder_that_follows_it() { + let map = map_with( + vec![MapEntry { + property: MapProperty::PayloadData(1), + property_value: "50% off %Cost%".to_string(), + values: vec![], + }], + vec![], + ); + + let mapped = apply_map(&map, &event()); + let value = &mapped.values[0]; + assert_eq!(value.unresolved, vec!["Cost".to_string()]); + assert!(!value.is_complete()); + assert_eq!(value.text, "50% off %Cost%"); + } + + #[test] + fn a_literal_percent_survives_when_a_later_placeholder_resolves() { + let map = map_with( + vec![MapEntry { + property: MapProperty::PayloadData(1), + property_value: "50% off for %user%".to_string(), + values: vec![binding("user", "SubjectUserName")], + }], + vec![], + ); + + let mapped = apply_map(&map, &event()); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("50% off for adam") + ); + assert!(mapped.values[0].is_complete()); + } + + #[test] + fn an_unpaired_trailing_percent_is_preserved_verbatim() { + let map = map_with( + vec![MapEntry { + property: MapProperty::PayloadData(1), + property_value: "complete: 100%".to_string(), + values: vec![], + }], + vec![], + ); + + let mapped = apply_map(&map, &event()); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("complete: 100%") + ); + assert!(mapped.values[0].is_complete()); + } + #[test] fn an_unused_binding_does_not_affect_the_result() { let map = map_with( diff --git a/crates/cmtraceopen-parser/src/eventmap/node.rs b/crates/cmtraceopen-parser/src/eventmap/node.rs index 76502541f..0bc5d6545 100644 --- a/crates/cmtraceopen-parser/src/eventmap/node.rs +++ b/crates/cmtraceopen-parser/src/eventmap/node.rs @@ -60,7 +60,13 @@ impl EventNode { } /// Returns the child elements called `name`, in document order. - pub fn children_named<'a>(&'a self, name: &'a str) -> impl Iterator { + /// + /// `name` carries its own lifetime so callers can pass a short-lived borrow, such as a field + /// of a path step being walked, without tying it to how long this node lives. + pub fn children_named<'a, 'n>( + &'a self, + name: &'n str, + ) -> impl Iterator + use<'a, 'n> { self.children.iter().filter(move |child| child.name == name) } } diff --git a/crates/cmtraceopen-parser/src/eventmap/path.rs b/crates/cmtraceopen-parser/src/eventmap/path.rs index 4aac2b1dd..f4dbcb0b5 100644 --- a/crates/cmtraceopen-parser/src/eventmap/path.rs +++ b/crates/cmtraceopen-parser/src/eventmap/path.rs @@ -217,12 +217,7 @@ fn parse_step(segment: &str, expression: &str) -> Result { } fn select_one<'a>(parent: &'a EventNode, step: &Step) -> Option<&'a EventNode> { - // Filtered inline rather than through EventNode::children_named so the step's lifetime stays - // independent of the node's; the iterator does not outlive this call. - let mut candidates = parent - .children - .iter() - .filter(|child| child.name == step.name); + let mut candidates = parent.children_named(&step.name); match &step.predicate { None => candidates.next(), Some(Predicate::Index(index)) => candidates.nth(index - 1), From 9ed4e3e554042563c27abc0501f1a915d6b018f8 Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 9 Aug 2026 14:13:35 -0400 Subject: [PATCH 05/85] feat(parser): build XPath queries so the service does the filtering Filtering happens either inside the Event Log service or in the client after every matching event has been fetched and rendered. FullEventLogView pushes only Event ID down and evaluates level, provider, time, and description client-side, which is why its default "last 7 days" costs a walk of every channel. Our live path currently sends the literal "*" and filters afterwards, so it has the same shape. This builds the query instead: relative and absolute time windows, levels, Event IDs with ranges, providers, and a keyword mask, include or exclude. It is pure string construction with no Windows dependency, so it lives in the parser crate and is unit-testable off Windows. Two constraints come from the service rather than from taste. Expression count is capped, so a large include list is split across several Select nodes inside one QueryList. Each node repeats the other predicates, because the service unions nodes rather than intersecting them; without that repetition the second node would match every level and silently widen the result. Exclusion lists are never split, since "not (a or b)" spread across unioned nodes becomes "not a or not b" and matches almost everything. Provider names reach this from user input and from event data, so interpolated values are escaped at both the XML and XPath layers. An unescaped apostrophe would terminate the string literal and let the remainder be read as query syntax; a test asserts "Evil' or '1'='1" cannot survive as syntax. Gates: 20 focused tests, clippy -D warnings clean, rustfmt clean, wasm32 target builds. Refs #539. Co-Authored-By: Claude Opus 5 --- .../cmtraceopen-parser/src/event_query/mod.rs | 475 ++++++++++++++++++ crates/cmtraceopen-parser/src/lib.rs | 1 + 2 files changed, 476 insertions(+) create mode 100644 crates/cmtraceopen-parser/src/event_query/mod.rs diff --git a/crates/cmtraceopen-parser/src/event_query/mod.rs b/crates/cmtraceopen-parser/src/event_query/mod.rs new file mode 100644 index 000000000..d6ba0f73b --- /dev/null +++ b/crates/cmtraceopen-parser/src/event_query/mod.rs @@ -0,0 +1,475 @@ +//! Building XPath queries for the Windows Event Log service. +//! +//! Filtering can happen in two places: inside the service, or in the client after every matching +//! event has been fetched and rendered. The difference is not marginal. FullEventLogView pushes +//! only Event ID down and evaluates level, provider, time, and description client-side, which is +//! why its default "last 7 days" costs a full walk of every channel. Reverse iteration plus an +//! early exit is what makes that survivable, not the filter itself. +//! +//! This module builds the query so the service does the work. It is pure string construction with +//! no Windows dependency, which keeps it inside the parser crate and unit-testable off Windows. +//! +//! Two constraints come from the service rather than from taste: +//! +//! - **Expression count is capped.** A query with too many `or` terms is rejected outright, so +//! large Event ID sets are split across several `` node. +/// +/// The service rejects a query whose expression count is too high. FullEventLogView splits at ten +/// per node, a bound reached by its own changelog after users hit the limit at twenty-three +/// expressions, so ten is used here too rather than rediscovering the ceiling in production. +const EVENT_IDS_PER_SELECT: usize = 10; + +/// Escapes a value for inclusion inside an XPath string literal within XML. +/// +/// Both layers matter. The XML layer would otherwise break on `&` or `<`, and the XPath layer +/// would break on the apostrophe that delimits the literal. +fn escape(value: &str) -> String { + let mut escaped = String::with_capacity(value.len()); + for character in value.chars() { + match character { + '&' => escaped.push_str("&"), + '<' => escaped.push_str("<"), + '>' => escaped.push_str(">"), + '"' => escaped.push_str("""), + // No XPath 1.0 escape exists for the delimiter, so it is dropped rather than allowed + // to terminate the literal. Provider names do not legitimately contain apostrophes. + '\'' => {} + _ => escaped.push(character), + } + } + escaped +} + +fn join_or(predicates: &[String]) -> String { + format!("({})", predicates.join(" or ")) +} + +fn time_predicate(window: &TimeWindow) -> Option { + match window { + TimeWindow::Last { milliseconds } => Some(format!( + "TimeCreated[timediff(@SystemTime) <= {milliseconds}]" + )), + TimeWindow::Between { from, to } => { + let mut bounds = Vec::new(); + if let Some(from) = from { + bounds.push(format!("@SystemTime >= '{}'", escape(from))); + } + if let Some(to) = to { + bounds.push(format!("@SystemTime <= '{}'", escape(to))); + } + if bounds.is_empty() { + return None; + } + Some(format!("TimeCreated[{}]", bounds.join(" and "))) + } + } +} + +fn system_predicates(filter: &EventQueryFilter, event_ids: &[EventIdSelector]) -> Vec { + let mut predicates = Vec::new(); + + if let Some(window) = filter.time.as_ref().and_then(time_predicate) { + predicates.push(window); + } + + if !filter.levels.is_empty() { + let levels: Vec = filter + .levels + .iter() + .map(|level| format!("Level={level}")) + .collect(); + predicates.push(join_or(&levels)); + } + + if !event_ids.is_empty() { + let ids: Vec = event_ids.iter().map(EventIdSelector::predicate).collect(); + let clause = join_or(&ids); + predicates.push(match filter.event_id_mode { + SelectorMode::Include => clause, + SelectorMode::Exclude => format!("not {clause}"), + }); + } + + if !filter.providers.is_empty() { + let providers: Vec = filter + .providers + .iter() + .map(|provider| format!("@Name='{}'", escape(provider))) + .collect(); + let clause = format!("Provider[{}]", providers.join(" or ")); + predicates.push(match filter.provider_mode { + SelectorMode::Include => clause, + SelectorMode::Exclude => format!("not {clause}"), + }); + } + + if let Some(keywords) = filter.keywords { + predicates.push(format!("band(Keywords,{keywords})")); + } + + predicates +} + +fn select_body(filter: &EventQueryFilter, event_ids: &[EventIdSelector]) -> String { + let predicates = system_predicates(filter, event_ids); + if predicates.is_empty() { + return "*".to_string(); + } + format!("*[System[{}]]", predicates.join(" and ")) +} + +/// Builds the query string passed to `EvtQuery`. +/// +/// Returns `*` when nothing is filtered. When the Event ID set is larger than one `{}", + select_body(filter, chunk) + ); + } + query.push_str(""); + query +} + +#[cfg(test)] +mod tests { + use super::*; + + fn filter() -> EventQueryFilter { + EventQueryFilter::default() + } + + #[test] + fn an_empty_filter_is_the_unfiltered_wildcard() { + assert!(filter().is_unfiltered()); + assert_eq!(build_query(&filter()), "*"); + } + + #[test] + fn a_relative_window_uses_timediff_against_the_service_clock() { + let mut f = filter(); + f.time = Some(TimeWindow::Last { + milliseconds: 604_800_000, + }); + assert_eq!( + build_query(&f), + "*[System[TimeCreated[timediff(@SystemTime) <= 604800000]]]" + ); + } + + #[test] + fn an_absolute_range_emits_both_bounds() { + let mut f = filter(); + f.time = Some(TimeWindow::Between { + from: Some("2026-08-01T00:00:00.000Z".into()), + to: Some("2026-08-09T00:00:00.000Z".into()), + }); + assert_eq!( + build_query(&f), + "*[System[TimeCreated[@SystemTime >= '2026-08-01T00:00:00.000Z' and @SystemTime <= '2026-08-09T00:00:00.000Z']]]" + ); + } + + #[test] + fn a_half_open_range_emits_only_the_bound_that_was_given() { + let mut f = filter(); + f.time = Some(TimeWindow::Between { + from: Some("2026-08-01T00:00:00.000Z".into()), + to: None, + }); + assert_eq!( + build_query(&f), + "*[System[TimeCreated[@SystemTime >= '2026-08-01T00:00:00.000Z']]]" + ); + } + + #[test] + fn a_range_with_no_bounds_contributes_nothing() { + let mut f = filter(); + f.time = Some(TimeWindow::Between { + from: None, + to: None, + }); + f.levels = vec![2]; + assert_eq!(build_query(&f), "*[System[(Level=2)]]"); + } + + #[test] + fn levels_are_unioned() { + let mut f = filter(); + f.levels = vec![1, 2, 3]; + assert_eq!( + build_query(&f), + "*[System[(Level=1 or Level=2 or Level=3)]]" + ); + } + + #[test] + fn event_ids_support_single_values_and_ranges() { + let mut f = filter(); + f.event_ids = vec![ + EventIdSelector::Single(4624), + EventIdSelector::Range(5000, 5010), + ]; + assert_eq!( + build_query(&f), + "*[System[(EventID=4624 or (EventID >= 5000 and EventID <= 5010))]]" + ); + } + + #[test] + fn a_reversed_range_is_normalized_rather_than_emitted_backwards() { + let mut f = filter(); + f.event_ids = vec![EventIdSelector::Range(9, 5)]; + assert_eq!( + build_query(&f), + "*[System[((EventID >= 5 and EventID <= 9))]]" + ); + } + + #[test] + fn a_degenerate_range_collapses_to_a_single_id() { + let mut f = filter(); + f.event_ids = vec![EventIdSelector::Range(7, 7)]; + assert_eq!(build_query(&f), "*[System[(EventID=7)]]"); + } + + #[test] + fn exclusion_negates_the_whole_clause() { + let mut f = filter(); + f.event_ids = vec![EventIdSelector::Single(4688)]; + f.event_id_mode = SelectorMode::Exclude; + assert_eq!(build_query(&f), "*[System[not (EventID=4688)]]"); + } + + #[test] + fn providers_are_matched_by_name() { + let mut f = filter(); + f.providers = vec!["Microsoft-Windows-Shell-Core".into()]; + assert_eq!( + build_query(&f), + "*[System[Provider[@Name='Microsoft-Windows-Shell-Core']]]" + ); + } + + #[test] + fn keywords_are_matched_with_band() { + let mut f = filter(); + f.keywords = Some(0x8000_0000_0000_0000); + assert_eq!( + build_query(&f), + "*[System[band(Keywords,9223372036854775808)]]" + ); + } + + #[test] + fn multiple_dimensions_are_intersected() { + let mut f = filter(); + f.time = Some(TimeWindow::Last { + milliseconds: 3_600_000, + }); + f.levels = vec![2]; + f.event_ids = vec![EventIdSelector::Single(1000)]; + f.providers = vec!["ESENT".into()]; + assert_eq!( + build_query(&f), + "*[System[TimeCreated[timediff(@SystemTime) <= 3600000] and (Level=2) and (EventID=1000) and Provider[@Name='ESENT']]]" + ); + } + + #[test] + fn a_large_id_set_is_split_across_select_nodes_in_one_query_list() { + let mut f = filter(); + f.event_ids = (1..=25).map(EventIdSelector::Single).collect(); + let query = build_query(&f); + + assert!(query.starts_with("")); + assert!(query.ends_with("")); + assert_eq!( + query.matches("").count(), 2); + assert_eq!(query.matches("(Level=2)").count(), 2); + } + + #[test] + fn an_exclusion_list_is_never_split_because_union_would_invert_it() { + // "not (a or b)" spread across unioned nodes becomes "not a or not b", which matches + // almost everything. Excludes stay in one node even when large. + let mut f = filter(); + f.event_ids = (1..=25).map(EventIdSelector::Single).collect(); + f.event_id_mode = SelectorMode::Exclude; + let query = build_query(&f); + + assert!(!query.contains("")); + assert!(query.starts_with("*[System[not (")); + } + + #[test] + fn a_set_exactly_at_the_bound_is_not_split() { + let mut f = filter(); + f.event_ids = (1..=EVENT_IDS_PER_SELECT as u32) + .map(EventIdSelector::Single) + .collect(); + assert!(!build_query(&f).contains("")); + } + + #[test] + fn an_apostrophe_cannot_terminate_a_string_literal() { + let mut f = filter(); + f.providers = vec!["Evil' or '1'='1".into()]; + let query = build_query(&f); + + assert!( + !query.contains("or '1'='1"), + "injected syntax must not survive: {query}" + ); + assert_eq!(query, "*[System[Provider[@Name='Evil or 1=1']]]"); + } + + #[test] + fn xml_metacharacters_in_a_provider_are_escaped() { + let mut f = filter(); + f.providers = vec!["A&BD\"E".into()]; + assert_eq!( + build_query(&f), + "*[System[Provider[@Name='A&B<C>D"E']]]" + ); + } + + #[test] + fn provider_exclusion_negates_the_clause() { + let mut f = filter(); + f.providers = vec!["Noisy-Provider".into()]; + f.provider_mode = SelectorMode::Exclude; + assert_eq!( + build_query(&f), + "*[System[not Provider[@Name='Noisy-Provider']]]" + ); + } +} diff --git a/crates/cmtraceopen-parser/src/lib.rs b/crates/cmtraceopen-parser/src/lib.rs index 06834eff0..848e6a497 100644 --- a/crates/cmtraceopen-parser/src/lib.rs +++ b/crates/cmtraceopen-parser/src/lib.rs @@ -11,6 +11,7 @@ pub mod collector; pub mod dsregcmd; pub mod error_db; pub mod esp; +pub mod event_query; pub mod eventmap; pub mod intune; pub mod models; From 297683389e6a23bae12b896c0c93eb35ec37374d Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 9 Aug 2026 14:17:25 -0400 Subject: [PATCH 06/85] perf(event-log): filter in the service and query channels concurrently Four changes to the live query path, all aimed at the complaint that started this work: a time window should cost a bounded query, not a walk of every channel. The query is now built rather than hardcoded. The path sent the literal "*" and filtered afterwards, which is the same shape as FullEventLogView and has the same cost. It now compiles the caller's filter to XPath via cmtraceopen_parser::event_query, so non-matching events are never fetched, rendered, or transferred. query_channel_filtered exposes this; the existing entry points keep their behaviour by passing an empty filter. EvtNext fetches 256 handles per call instead of 16. Each call is a round trip to the service and is the dominant cost of a scan. The API accepts up to 1024; 256 cuts round trips by 16x against the previous value while keeping the per-call array modest. FullEventLogView hardcodes 1. EvtQueryTolerateQueryErrors is now set. Without it a single element the service cannot evaluate, such as a provider not registered on this machine, aborts the whole channel and the result silently looks empty rather than partial. Channels are queried concurrently with rayon. Each channel is an independent conversation that spends nearly all its time waiting on RPC, so serializing them left the machine idle. Results are collected per channel and ordered afterwards, so concurrency cannot affect output. This is structurally impossible for FullEventLogView, which imports no threading primitives at all. A failed channel still reports 0 events so the coverage gap stays visible instead of reading as a channel that had nothing in it. Windows-only code, so compilation is verified by the Windows CI job rather than locally; measurement against the benchmark gate still has to happen on the lab host. macOS side: clippy -D warnings clean, rustfmt clean, 1,095 tests pass, parser crate still builds for wasm32. Refs #539. Co-Authored-By: Claude Opus 5 --- src-tauri/src/event_log/commands.rs | 46 ++++++++++++++++------- src-tauri/src/event_log/live.rs | 58 +++++++++++++++++++++++++++-- 2 files changed, 87 insertions(+), 17 deletions(-) diff --git a/src-tauri/src/event_log/commands.rs b/src-tauri/src/event_log/commands.rs index 6e797c330..ea6f4e497 100644 --- a/src-tauri/src/event_log/commands.rs +++ b/src-tauri/src/event_log/commands.rs @@ -45,23 +45,42 @@ pub async fn evtx_query_channels( #[cfg(target_os = "windows")] { tokio::task::spawn_blocking(move || { + use rayon::prelude::*; + + // Channels are queried concurrently. Each one is an independent conversation with the + // Event Log service that spends nearly all its time waiting on RPC, so serializing + // them leaves the machine idle. Results are collected per channel and ordered + // afterwards, so concurrency cannot affect the output. + let per_channel: Vec<(String, Result, String>)> = + channels + .par_iter() + .map(|channel| { + let app_ref = &app; + let ch_name = channel.clone(); + let outcome = super::live::query_channel_with_progress( + channel, + max_events, + |fetched, _| { + let _ = app_ref.emit( + "evtx-query-progress", + EvtxQueryProgress { + channel: ch_name.clone(), + fetched, + }, + ); + }, + ); + (channel.clone(), outcome) + }) + .collect(); + let mut all_records = Vec::new(); let mut channel_infos = Vec::new(); let mut parse_errors = 0u32; let mut error_messages = Vec::new(); - for channel in &channels { - let app_ref = &app; - let ch_name = channel.clone(); - match super::live::query_channel_with_progress(channel, max_events, |fetched, _| { - let _ = app_ref.emit( - "evtx-query-progress", - EvtxQueryProgress { - channel: ch_name.clone(), - fetched, - }, - ); - }) { + for (channel, outcome) in per_channel { + match outcome { Ok(records) => { channel_infos.push(super::models::EvtxChannelInfo { name: channel.clone(), @@ -77,7 +96,8 @@ pub async fn evtx_query_channels( e ); error_messages.push(format!("{}: {}", channel, e)); - // Still include channel in results with 0 events so frontend knows it was attempted + // A failed channel is still reported with 0 events so the gap stays visible + // rather than looking like a channel that simply had nothing in it. channel_infos.push(super::models::EvtxChannelInfo { name: channel.clone(), event_count: 0, diff --git a/src-tauri/src/event_log/live.rs b/src-tauri/src/event_log/live.rs index 7835b9c44..75375c703 100644 --- a/src-tauri/src/event_log/live.rs +++ b/src-tauri/src/event_log/live.rs @@ -6,15 +6,25 @@ use regex::Regex; use super::models::{ChannelSourceType, EvtxChannelInfo, EvtxField, EvtxLevel, EvtxRecord}; use super::sanitize_control_chars; +use cmtraceopen_parser::event_query::{build_query, EventQueryFilter}; #[cfg(target_os = "windows")] use windows::core::{Error, HSTRING, PCWSTR}; #[cfg(target_os = "windows")] use windows::Win32::System::EventLog::{ EvtClose, EvtFormatMessage, EvtFormatMessageEvent, EvtNext, EvtOpenPublisherMetadata, EvtQuery, - EvtQueryChannelPath, EvtQueryReverseDirection, EvtRender, EvtRenderEventXml, EVT_HANDLE, + EvtQueryChannelPath, EvtQueryReverseDirection, EvtQueryTolerateQueryErrors, EvtRender, + EvtRenderEventXml, EVT_HANDLE, }; +/// Event handles fetched per `EvtNext` call. +/// +/// Each call is a round trip to the Event Log service, so this is the dominant cost of a scan. +/// FullEventLogView hardcodes 1, paying one round trip per event. The API accepts up to 1024; +/// 256 keeps the per-call array modest while cutting round trips by that factor. +#[cfg(target_os = "windows")] +const EVENT_FETCH_BATCH: usize = 256; + // ── RAII handle wrapper ───────────────────────────────────────────────────── #[cfg(target_os = "windows")] @@ -119,23 +129,54 @@ pub fn query_channel(channel: &str, max_events: Option) -> Result, +) -> Result, String> { + query_channel_inner(channel, filter, max_events, |_, _| {}) +} + /// Query with a progress callback: `on_progress(fetched_so_far, total_estimate)`. #[cfg(target_os = "windows")] pub fn query_channel_with_progress( channel: &str, max_events: Option, on_progress: impl Fn(usize, Option), +) -> Result, String> { + query_channel_inner( + channel, + &EventQueryFilter::default(), + max_events, + on_progress, + ) +} + +#[cfg(target_os = "windows")] +fn query_channel_inner( + channel: &str, + filter: &EventQueryFilter, + max_events: Option, + on_progress: impl Fn(usize, Option), ) -> Result, String> { let limit = max_events.map(|n| n as usize).unwrap_or(usize::MAX); let channel_hstring = HSTRING::from(channel); - let query_string = HSTRING::from("*"); + let query_string = HSTRING::from(build_query(filter).as_str()); let query_handle = unsafe { EvtQuery( None, &channel_hstring, &query_string, - EvtQueryChannelPath.0 | EvtQueryReverseDirection.0, + // TolerateQueryErrors keeps a scan alive when one part of a query cannot be evaluated, + // for example a provider that is not registered on this machine. Without it a single + // bad element aborts the whole channel and the result silently looks empty. + EvtQueryChannelPath.0 | EvtQueryReverseDirection.0 | EvtQueryTolerateQueryErrors.0, ) } .map_err(|e| format_error(&format!("EvtQuery({channel})"), &e))?; @@ -146,7 +187,7 @@ pub fn query_channel_with_progress( let mut publisher_metadata = HashMap::>::new(); while records.len() < limit { - let mut raw_handles = [0isize; 16]; + let mut raw_handles = [0isize; EVENT_FETCH_BATCH]; let mut returned = 0u32; match unsafe { EvtNext(query_handle.raw(), &mut raw_handles, 0, 0, &mut returned) } { @@ -234,6 +275,15 @@ pub fn query_channel(_channel: &str, _max_events: Option) -> Result, +) -> Result, String> { + Err("Live event log queries are only available on Windows.".to_string()) +} + // ── Win32 helpers (Windows only) ──────────────────────────────────────────── #[cfg(target_os = "windows")] From c91d4fc891f26f77600ef78fbbb897381dcfad8d Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 9 Aug 2026 14:22:15 -0400 Subject: [PATCH 07/85] feat(event-log): give the live view a server-side time window Completes the path from the UI to the Event Log service so the query work is actually reachable. The filter now crosses IPC and reaches EvtQuery as XPath. EventQueryFilter gains serde derives with a camelCase wire shape, so the same type is the Rust API and the TypeScript contract, and a round-trip test asserts the compiled query is identical on both sides of the boundary. Every field defaults, because the frontend sends only what the operator set. The live view gains a time window control offering the last hour, 24 hours, 7 days, 30 days, or all time, defaulting to 24 hours. That default is the point of the exercise: FullEventLogView loads seven days on startup and filters time client-side, so the window costs a walk of every channel. Here the window is a service-side predicate, so events outside it are never fetched, rendered, or transferred, and widening it to 30 days is cheap rather than punitive. Changing the window refetches, since a server-side predicate cannot be applied to records already in memory. The control only appears for live sources, where it means something. Level and provider deliberately stay client-side for now. Their existing controls filter records already loaded, and moving them server-side changes what a reload fetches. That is a behavioural change worth making deliberately rather than smuggling in alongside this one. Gates: tsc clean, clippy -D warnings clean on both crates, rustfmt clean, wasm32 target builds. Refs #539. Co-Authored-By: Claude Opus 5 --- .../cmtraceopen-parser/src/event_query/mod.rs | 96 +++++++++++++++---- src-tauri/src/event_log/commands.rs | 10 +- src-tauri/src/event_log/live.rs | 21 ++++ src/workspaces/event-log/EvtxFilterBar.tsx | 44 ++++++++- src/workspaces/event-log/evtx-store.ts | 22 +++++ src/workspaces/event-log/types.ts | 31 ++++++ 6 files changed, 202 insertions(+), 22 deletions(-) diff --git a/crates/cmtraceopen-parser/src/event_query/mod.rs b/crates/cmtraceopen-parser/src/event_query/mod.rs index d6ba0f73b..019941d31 100644 --- a/crates/cmtraceopen-parser/src/event_query/mod.rs +++ b/crates/cmtraceopen-parser/src/event_query/mod.rs @@ -20,8 +20,11 @@ use std::fmt::Write as _; +use serde::{Deserialize, Serialize}; + /// How a set of values narrows a result set. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] pub enum SelectorMode { /// Only events matching the set. #[default] @@ -31,20 +34,21 @@ pub enum SelectorMode { } /// One Event ID, or an inclusive range of them. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", tag = "kind")] pub enum EventIdSelector { /// A single ID. - Single(u32), + Single { id: u32 }, /// An inclusive `low..=high` range. - Range(u32, u32), + Range { low: u32, high: u32 }, } impl EventIdSelector { fn predicate(&self) -> String { match self { - Self::Single(id) => format!("EventID={id}"), - Self::Range(low, high) if low == high => format!("EventID={low}"), - Self::Range(low, high) => { + Self::Single { id } => format!("EventID={id}"), + Self::Range { low, high } if low == high => format!("EventID={low}"), + Self::Range { low, high } => { let (low, high) = if low <= high { (low, high) } else { @@ -57,7 +61,8 @@ impl EventIdSelector { } /// The time span an event must fall within. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", tag = "kind")] pub enum TimeWindow { /// Everything newer than `milliseconds` ago, evaluated by the service against its own clock. /// @@ -72,7 +77,8 @@ pub enum TimeWindow { } /// Everything that can be pushed into the service instead of filtered afterwards. -#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] pub struct EventQueryFilter { /// Time span, if any. pub time: Option, @@ -318,8 +324,11 @@ mod tests { fn event_ids_support_single_values_and_ranges() { let mut f = filter(); f.event_ids = vec![ - EventIdSelector::Single(4624), - EventIdSelector::Range(5000, 5010), + EventIdSelector::Single { id: 4624 }, + EventIdSelector::Range { + low: 5000, + high: 5010, + }, ]; assert_eq!( build_query(&f), @@ -330,7 +339,7 @@ mod tests { #[test] fn a_reversed_range_is_normalized_rather_than_emitted_backwards() { let mut f = filter(); - f.event_ids = vec![EventIdSelector::Range(9, 5)]; + f.event_ids = vec![EventIdSelector::Range { low: 9, high: 5 }]; assert_eq!( build_query(&f), "*[System[((EventID >= 5 and EventID <= 9))]]" @@ -340,14 +349,14 @@ mod tests { #[test] fn a_degenerate_range_collapses_to_a_single_id() { let mut f = filter(); - f.event_ids = vec![EventIdSelector::Range(7, 7)]; + f.event_ids = vec![EventIdSelector::Range { low: 7, high: 7 }]; assert_eq!(build_query(&f), "*[System[(EventID=7)]]"); } #[test] fn exclusion_negates_the_whole_clause() { let mut f = filter(); - f.event_ids = vec![EventIdSelector::Single(4688)]; + f.event_ids = vec![EventIdSelector::Single { id: 4688 }]; f.event_id_mode = SelectorMode::Exclude; assert_eq!(build_query(&f), "*[System[not (EventID=4688)]]"); } @@ -379,7 +388,7 @@ mod tests { milliseconds: 3_600_000, }); f.levels = vec![2]; - f.event_ids = vec![EventIdSelector::Single(1000)]; + f.event_ids = vec![EventIdSelector::Single { id: 1000 }]; f.providers = vec!["ESENT".into()]; assert_eq!( build_query(&f), @@ -390,7 +399,7 @@ mod tests { #[test] fn a_large_id_set_is_split_across_select_nodes_in_one_query_list() { let mut f = filter(); - f.event_ids = (1..=25).map(EventIdSelector::Single).collect(); + f.event_ids = (1..=25).map(|id| EventIdSelector::Single { id }).collect(); let query = build_query(&f); assert!(query.starts_with("")); @@ -410,7 +419,7 @@ mod tests { // would match every level and silently widen the result set. let mut f = filter(); f.levels = vec![2]; - f.event_ids = (1..=15).map(EventIdSelector::Single).collect(); + f.event_ids = (1..=15).map(|id| EventIdSelector::Single { id }).collect(); let query = build_query(&f); assert_eq!(query.matches(" setFilterSearch(data.value)} diff --git a/src/workspaces/event-log/EvtxTimeline.tsx b/src/workspaces/event-log/EvtxTimeline.tsx index 79fd8f666..35587e914 100644 --- a/src/workspaces/event-log/EvtxTimeline.tsx +++ b/src/workspaces/event-log/EvtxTimeline.tsx @@ -7,6 +7,7 @@ import { } from "../../lib/log-accessibility"; import { useUiStore } from "../../stores/ui-store"; import { useEvtxStore, type EvtxSortField } from "./evtx-store"; +import { parseEventIdFilter } from "./evtx-filter"; import type { EvtxRecord, EvtxLevel } from "./types"; import { EvtxTimelineRow } from "./EvtxTimelineRow"; @@ -45,17 +46,6 @@ function compareRecords( return direction === "asc" ? cmp : -cmp; } -function parseEventIdFilter(raw: string): Set | null { - const trimmed = raw.trim(); - if (!trimmed) return null; - const ids = new Set(); - for (const part of trimmed.split(",")) { - const n = parseInt(part.trim(), 10); - if (!isNaN(n)) ids.add(n); - } - return ids.size > 0 ? ids : null; -} - export function EvtxTimeline() { const records = useEvtxStore((s) => s.records); const selectedChannels = useEvtxStore((s) => s.selectedChannels); diff --git a/src/workspaces/event-log/evtx-filter.test.ts b/src/workspaces/event-log/evtx-filter.test.ts new file mode 100644 index 000000000..7b225cf35 --- /dev/null +++ b/src/workspaces/event-log/evtx-filter.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; +import { parseEventIdFilter } from "./evtx-filter"; + +describe("parseEventIdFilter", () => { + it("returns null when the box constrains nothing", () => { + expect(parseEventIdFilter("")).toBeNull(); + expect(parseEventIdFilter(" ")).toBeNull(); + }); + + it("parses a comma separated list", () => { + expect(parseEventIdFilter("4624,4625")).toEqual(new Set([4624, 4625])); + }); + + it("tolerates spaces as separators and around commas", () => { + expect(parseEventIdFilter("4624 4625")).toEqual(new Set([4624, 4625])); + expect(parseEventIdFilter(" 4624 , 4625 ")).toEqual(new Set([4624, 4625])); + }); + + it("expands an inclusive range", () => { + expect(parseEventIdFilter("5-8")).toEqual(new Set([5, 6, 7, 8])); + }); + + it("normalizes a reversed range rather than yielding nothing", () => { + expect(parseEventIdFilter("8-5")).toEqual(new Set([5, 6, 7, 8])); + }); + + it("mixes singles and ranges", () => { + expect(parseEventIdFilter("1, 4-6, 9")).toEqual(new Set([1, 4, 5, 6, 9])); + }); + + it("ignores tokens that are not ids instead of failing the whole filter", () => { + // A half-typed filter should narrow by what is parseable, not silently match everything. + expect(parseEventIdFilter("4624, abc")).toEqual(new Set([4624])); + }); + + it("returns null when nothing in the box parsed", () => { + expect(parseEventIdFilter("abc, def")).toBeNull(); + }); +}); diff --git a/src/workspaces/event-log/evtx-filter.ts b/src/workspaces/event-log/evtx-filter.ts new file mode 100644 index 000000000..7d614ca5c --- /dev/null +++ b/src/workspaces/event-log/evtx-filter.ts @@ -0,0 +1,67 @@ +/** + * Pure filtering helpers for the event log view. + * + * Deliberately separate from `evtx-store.ts`, which subscribes to Tauri events at module scope. + * Importing the store from a test fires that subscription, so anything worth unit-testing lives + * here where it can be imported without a Tauri runtime. + */ +import type { EvtxLevel, EvtxRecord } from "./types"; + +/** + * Parses the Event ID filter box into a set, or null when it constrains nothing. + * + * Accepts comma or space separated ids and inclusive `low-high` ranges, matching what operators + * expect from the incumbent tools. + */ +export function parseEventIdFilter(raw: string): Set | null { + const trimmed = raw.trim(); + if (!trimmed) return null; + const ids = new Set(); + for (const token of trimmed.split(/[\s,]+/).filter(Boolean)) { + const range = token.match(/^(\d+)-(\d+)$/); + if (range) { + const low = Number(range[1]); + const high = Number(range[2]); + const [from, to] = low <= high ? [low, high] : [high, low]; + for (let id = from; id <= to; id += 1) ids.add(id); + continue; + } + const single = Number(token); + if (Number.isInteger(single)) ids.add(single); + } + return ids.size > 0 ? ids : null; +} + +/** The subset of store state that decides which records are on screen. */ +export interface VisibleRecordsInput { + records: EvtxRecord[]; + selectedChannels: Set; + filterLevels: Set; + filterEventIds: string; + filterSearch: string; +} + +/** + * The records currently on screen, before sorting. + * + * Shared so an export writes exactly what the operator is looking at. Recomputing the predicate at + * the export site would let the two drift, and an export that quietly differs from the view is + * worse than no export at all. + */ +export function selectVisibleRecords(input: VisibleRecordsInput): EvtxRecord[] { + const eventIdSet = parseEventIdFilter(input.filterEventIds); + const search = input.filterSearch.trim().toLowerCase(); + return input.records.filter((r) => { + if (!input.selectedChannels.has(r.channel)) return false; + if (!input.filterLevels.has(r.level)) return false; + if (eventIdSet && !eventIdSet.has(r.eventId)) return false; + if ( + search && + !r.message.toLowerCase().includes(search) && + !r.provider.toLowerCase().includes(search) + ) { + return false; + } + return true; + }); +} diff --git a/src/workspaces/event-log/evtx-store.ts b/src/workspaces/event-log/evtx-store.ts index 1ac2d30ed..7347117d4 100644 --- a/src/workspaces/event-log/evtx-store.ts +++ b/src/workspaces/event-log/evtx-store.ts @@ -11,6 +11,9 @@ import type { } from "./types"; import { EVTX_TIME_WINDOW_MS } from "./types"; +// Re-exported so callers have one import site; the implementations live in a Tauri-free module. +export { parseEventIdFilter, selectVisibleRecords } from "./evtx-filter"; + /** * Builds the filter handed to the backend, which compiles it to XPath. * @@ -379,3 +382,4 @@ listen<{ channel: string; fetched: number }>("evtx-query-progress", (event) => { loadingProgress: event.payload.fetched, }); }); + From 4ec7066166051fecf5f92a2110f921619958fded Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 9 Aug 2026 14:49:15 -0400 Subject: [PATCH 12/85] feat(event-log): apply loaded maps to events and surface the columns Closes the loop on the map engine. Maps could be parsed and applied in isolation, but nothing loaded them into a running app or put the results in front of anyone. The registry is process-wide rather than threaded through every call. Both the live and file record paths already parse each event once for the System block, so applying maps at that point costs no second parse. An empty registry is the normal state until maps are loaded and simply yields no columns. Records gain a mapped list, and the details pane renders it only when a map covers that event type, so no empty heading appears for the overwhelming majority of events that have no map. A column whose map referenced a field the event did not carry is marked incomplete and shown in warning colour with its unresolved placeholder intact, rather than being quietly blanked. Two commands are added: one to load a directory of maps and report what loaded, what was superseded, and what failed, and one to report how many are in effect. An operator who wonders why an event type is not being mapped can see the answer instead of guessing. An end-to-end test proves the whole chain: YAML on disk, into the registry, applied to XML parsed by the host adapter, out as columns, with a non-matching event id confirming no map means no columns rather than a guess. Gates: 62 event_log tests pass, full src-tauri suite green, clippy -D warnings clean, rustfmt clean, tsc clean, vitest exits 0. Refs #539. Co-Authored-By: Claude Opus 5 --- src-tauri/src/event_log/commands.rs | 20 +++ src-tauri/src/event_log/export.rs | 1 + src-tauri/src/event_log/live.rs | 13 +- src-tauri/src/event_log/maps.rs | 145 +++++++++++++++++++- src-tauri/src/event_log/models.rs | 3 + src-tauri/src/event_log/parser.rs | 13 +- src-tauri/src/lib.rs | 2 + src/workspaces/event-log/EvtxDetailPane.tsx | 51 +++++++ src/workspaces/event-log/types.ts | 9 ++ 9 files changed, 249 insertions(+), 8 deletions(-) diff --git a/src-tauri/src/event_log/commands.rs b/src-tauri/src/event_log/commands.rs index b427035be..998ec5502 100644 --- a/src-tauri/src/event_log/commands.rs +++ b/src-tauri/src/event_log/commands.rs @@ -167,3 +167,23 @@ pub async fn evtx_export_records( ); Ok(byte_count) } + +/// Loads EvtxECmd `.map` files from `directory` into the process registry. +/// +/// Returns what loaded, what was superseded, and what failed, so an operator can see why an event +/// type is not being mapped rather than being left guessing. +#[tauri::command] +pub async fn evtx_load_event_maps( + directory: String, +) -> Result { + let path = std::path::PathBuf::from(&directory); + tokio::task::spawn_blocking(move || super::maps::load_global(&path)) + .await + .map_err(|error| format!("map load task failed: {error}"))? +} + +/// Number of maps currently in effect. +#[tauri::command] +pub async fn evtx_loaded_map_count() -> u64 { + super::maps::loaded_count() as u64 +} diff --git a/src-tauri/src/event_log/export.rs b/src-tauri/src/event_log/export.rs index 19b7e18b9..612de8844 100644 --- a/src-tauri/src/event_log/export.rs +++ b/src-tauri/src/event_log/export.rs @@ -184,6 +184,7 @@ mod tests { thread_id: None, user_sid: Some("S-1-5-18".into()), keywords: Some("0x80".into()), + mapped: Vec::new(), } } diff --git a/src-tauri/src/event_log/live.rs b/src-tauri/src/event_log/live.rs index a676a097e..09b0d8743 100644 --- a/src-tauri/src/event_log/live.rs +++ b/src-tauri/src/event_log/live.rs @@ -423,8 +423,16 @@ fn parse_xml_to_record( .map(sanitize_control_chars) .unwrap_or_else(|| build_event_data_summary(&event_data)); - let system = super::event_node::parse_event_xml(xml) - .map(|root| super::event_node::extract_system_fields(&root)) + // Parsed once and used for both the System block and any registered map, so mapping costs no + // extra parse. + let parsed = super::event_node::parse_event_xml(xml).ok(); + let system = parsed + .as_ref() + .map(super::event_node::extract_system_fields) + .unwrap_or_default(); + let mapped = parsed + .as_ref() + .map(|root| super::maps::apply_global(channel, &provider, event_id, root)) .unwrap_or_default(); Some(EvtxRecord { @@ -447,6 +455,7 @@ fn parse_xml_to_record( thread_id: system.thread_id, user_sid: system.user_sid, keywords: system.keywords, + mapped, }) } diff --git a/src-tauri/src/event_log/maps.rs b/src-tauri/src/event_log/maps.rs index ffe490cb0..b53ee3bf5 100644 --- a/src-tauri/src/event_log/maps.rs +++ b/src-tauri/src/event_log/maps.rs @@ -18,9 +18,10 @@ use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; +use std::sync::{OnceLock, RwLock}; -use cmtraceopen_parser::eventmap::{EventMap, MapRegistry}; -use serde::Serialize; +use cmtraceopen_parser::eventmap::{apply_map, EventMap, EventNode, MapProperty, MapRegistry}; +use serde::{Deserialize, Serialize}; /// Why a `.map` file could not be used. #[derive(Debug, Clone, Serialize, PartialEq, Eq)] @@ -167,7 +168,7 @@ mod tests { use std::fs; /// A real upstream map, byte for byte, including the leading comment style and quoting. - const SHELL_CORE_9701: &str = r#"Author: Troy Larson + pub(super) const SHELL_CORE_9701: &str = r#"Author: Troy Larson Description: RunOnceEx commands started EventId: 9701 Channel: Microsoft-Windows-Shell-Core/Operational @@ -298,3 +299,141 @@ Maps: assert!(load_maps_from_dir(&missing).is_err()); } } + +// ── Process-wide registry ─────────────────────────────────────────────────── + +/// The maps in effect for this process. +/// +/// Held globally rather than threaded through every call site because both the live and file +/// record paths already parse the event once and applying maps there avoids a second parse. An +/// empty registry is the normal state until maps are loaded, and it simply yields no columns. +fn global() -> &'static RwLock { + static REGISTRY: OnceLock> = OnceLock::new(); + REGISTRY.get_or_init(|| RwLock::new(MapRegistry::new())) +} + +/// One normalized column produced by a map. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct MappedColumn { + /// Column name, for example `UserName` or `PayloadData1`. + pub property: String, + /// The rendered text. + pub text: String, + /// False when the map referenced a field this event did not carry, in which case `text` still + /// contains the unresolved `%placeholder%`. + pub complete: bool, +} + +fn property_name(property: &MapProperty) -> String { + match property { + MapProperty::UserName => "UserName".to_string(), + MapProperty::RemoteHost => "RemoteHost".to_string(), + MapProperty::ExecutableInfo => "ExecutableInfo".to_string(), + MapProperty::PayloadData(slot) => format!("PayloadData{slot}"), + MapProperty::Other(name) => name.clone(), + } +} + +/// Replaces the process registry with the maps in `directory`. +pub fn load_global(directory: &Path) -> Result { + let (registry, outcome) = load_maps_from_dir(directory)?; + *global() + .write() + .map_err(|_| "map registry lock was poisoned".to_string())? = registry; + Ok(outcome) +} + +/// Number of maps currently loaded. +pub fn loaded_count() -> usize { + global().read().map(|registry| registry.len()).unwrap_or(0) +} + +/// Applies the registered map for this event, if one exists. +/// +/// Returns an empty vector when no map matches, which is the common case: the upstream corpus +/// covers a few hundred event types out of many thousands. +pub fn apply_global( + channel: &str, + provider: &str, + event_id: u32, + event: &EventNode, +) -> Vec { + let Ok(registry) = global().read() else { + return Vec::new(); + }; + let Some(map) = registry.find(channel, provider, event_id) else { + return Vec::new(); + }; + apply_map(map, event) + .values + .into_iter() + .map(|value| MappedColumn { + property: property_name(&value.property), + complete: value.unresolved.is_empty(), + text: value.text, + }) + .collect() +} + +#[cfg(test)] +mod global_tests { + use super::tests::SHELL_CORE_9701; + use super::*; + + #[test] + fn an_unloaded_registry_yields_no_columns_rather_than_failing() { + let event = EventNode::new("Event"); + // Whatever other tests have loaded, an event with no matching map must map to nothing. + assert!(apply_global("No-Such-Channel", "No-Such-Provider", 1, &event).is_empty()); + } + + #[test] + fn a_loaded_map_produces_columns_for_a_matching_event_end_to_end() { + // Proves the whole chain: YAML on disk, into the process registry, applied to XML parsed + // by the host adapter, out as columns the UI can render. + let dir = std::env::temp_dir().join("cmtraceopen-maps-global-e2e"); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("temp dir"); + fs::write(dir.join("shell-core-9701.map"), SHELL_CORE_9701).expect("write map"); + + let outcome = load_global(&dir).expect("loads"); + assert_eq!(outcome.loaded_count(), 1); + assert!(loaded_count() >= 1); + + let event = crate::event_log::event_node::parse_event_xml( + "RunOnceEx started", + ) + .expect("parses"); + + let columns = apply_global( + "Microsoft-Windows-Shell-Core/Operational", + "Microsoft-Windows-Shell-Core", + 9701, + &event, + ); + assert_eq!(columns.len(), 1); + assert_eq!(columns[0].property, "PayloadData1"); + assert_eq!(columns[0].text, "RunOnceEx started"); + assert!(columns[0].complete); + + // A different event id on the same channel has no map and must map to nothing. + assert!(apply_global( + "Microsoft-Windows-Shell-Core/Operational", + "Microsoft-Windows-Shell-Core", + 9702, + &event + ) + .is_empty()); + } + + #[test] + fn property_names_match_the_upstream_column_names() { + assert_eq!(property_name(&MapProperty::UserName), "UserName"); + assert_eq!(property_name(&MapProperty::PayloadData(3)), "PayloadData3"); + assert_eq!( + property_name(&MapProperty::Other("Custom".into())), + "Custom" + ); + } +} diff --git a/src-tauri/src/event_log/models.rs b/src-tauri/src/event_log/models.rs index 84a34fd13..f8f896aa0 100644 --- a/src-tauri/src/event_log/models.rs +++ b/src-tauri/src/event_log/models.rs @@ -37,6 +37,9 @@ pub struct EvtxRecord { /// Keyword bitmask as written by the provider, for example `0x8020000000000000`. #[serde(default)] pub keywords: Option, + /// Columns produced by an EvtxECmd map, empty when no map covers this event type. + #[serde(default)] + pub mapped: Vec, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/src-tauri/src/event_log/parser.rs b/src-tauri/src/event_log/parser.rs index 9ccb7cf3c..4ac496128 100644 --- a/src-tauri/src/event_log/parser.rs +++ b/src-tauri/src/event_log/parser.rs @@ -159,9 +159,15 @@ fn parse_single_file(path: &Path) -> Result<(Vec, u32), String> { let raw_xml = serde_json::to_string_pretty(json).unwrap_or_default(); // Parsed once from the raw XML so both the live and file paths read the System block the - // same way, rather than each growing its own extraction. - let system = super::event_node::parse_event_xml(&raw_xml) - .map(|root| super::event_node::extract_system_fields(&root)) + // same way, and so any registered map is applied without a second parse. + let parsed = super::event_node::parse_event_xml(&raw_xml).ok(); + let system = parsed + .as_ref() + .map(super::event_node::extract_system_fields) + .unwrap_or_default(); + let mapped = parsed + .as_ref() + .map(|root| super::maps::apply_global(&channel, &provider, event_id, root)) .unwrap_or_default(); records.push(EvtxRecord { id: 0, // Will be reassigned after sorting @@ -183,6 +189,7 @@ fn parse_single_file(path: &Path) -> Result<(Vec, u32), String> { thread_id: system.thread_id, user_sid: system.user_sid, keywords: system.keywords, + mapped, }); } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 63426537f..a981ea0ca 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -354,6 +354,8 @@ pub fn run() { #[cfg(feature = "event-log")] event_log::commands::evtx_query_channels, event_log::commands::evtx_export_records, + event_log::commands::evtx_load_event_maps, + event_log::commands::evtx_loaded_map_count, #[cfg(target_os = "windows")] commands::graph_api::graph_authenticate, #[cfg(target_os = "windows")] diff --git a/src/workspaces/event-log/EvtxDetailPane.tsx b/src/workspaces/event-log/EvtxDetailPane.tsx index 089a6ab69..415699ba4 100644 --- a/src/workspaces/event-log/EvtxDetailPane.tsx +++ b/src/workspaces/event-log/EvtxDetailPane.tsx @@ -232,6 +232,57 @@ export function EvtxDetailPane() { )} + {/* Map-derived columns. Only present where a map covers this event type, so the section is + hidden entirely rather than showing an empty heading. */} + {record.mapped && record.mapped.length > 0 && ( +
+
+ Mapped fields +
+ {record.mapped.map((column) => ( +
+ + {column.property} + + + {column.text} + +
+ ))} +
+ )} + {/* Raw XML */}
+ + + )} + + + ); + })} + + + s.groupBy); const collapsedGroups = useEvtxStore((s) => s.collapsedGroups); const toggleGroup = useEvtxStore((s) => s.toggleGroup); + const columnConfig = useEvtxStore((s) => s.columnConfig); const selectedRecordId = useEvtxStore((s) => s.selectedRecordId); const setSelectedRecordId = useEvtxStore((s) => s.setSelectedRecordId); @@ -301,6 +302,7 @@ export function EvtxTimeline() { smallFontSize={smallFontSize} monoFontSize={monoFontSize} lineHeight={lineHeight} + columnConfig={columnConfig} onSelect={setSelectedRecordId} /> ); diff --git a/src/workspaces/event-log/EvtxTimelineRow.tsx b/src/workspaces/event-log/EvtxTimelineRow.tsx index 27f6baa55..8d225103b 100644 --- a/src/workspaces/event-log/EvtxTimelineRow.tsx +++ b/src/workspaces/event-log/EvtxTimelineRow.tsx @@ -4,6 +4,12 @@ import { LOG_MONOSPACE_FONT_FAMILY, } from "../../lib/log-accessibility"; import type { EvtxRecord, EvtxLevel } from "./types"; +import { + columnValue, + columnWidth, + visibleColumns, + type EvtxColumnConfig, +} from "./evtx-columns"; const LEVEL_COLORS: Record = { Critical: tokens.colorPaletteRedForeground1, @@ -29,6 +35,7 @@ export interface EvtxTimelineRowProps { smallFontSize: number; monoFontSize: number; lineHeight: string; + columnConfig: EvtxColumnConfig; onSelect: (id: number | null) => void; } @@ -42,6 +49,7 @@ export const EvtxTimelineRow = memo( smallFontSize, monoFontSize, lineHeight, + columnConfig, onSelect, }, ref @@ -82,105 +90,70 @@ export const EvtxTimelineRow = memo( minWidth: 0, }} > - {/* Level badge */} -
- {LEVEL_SHORT[record.level]} -
- - {/* Timestamp */} -
- {record.timestamp} -
+ {visibleColumns(columnConfig).map((column) => { + const width = columnWidth(columnConfig, column); + const value = columnValue(record, column.id); - {/* Event ID */} -
- {record.eventId} -
- - {/* Channel badge */} -
- {record.channel} -
+ if (column.id === "level") { + return ( +
+ {LEVEL_SHORT[record.level]} +
+ ); + } - {/* Provider */} -
- {record.provider} -
+ const isDescription = column.id === "message"; + const isMono = column.id === "timestamp" || column.id === "keywords"; - {/* Message preview */} -
- {record.message} -
+ return ( +
+ {value} +
+ ); + })}
); }) diff --git a/src/workspaces/event-log/evtx-store.ts b/src/workspaces/event-log/evtx-store.ts index d34944627..2c5288cae 100644 --- a/src/workspaces/event-log/evtx-store.ts +++ b/src/workspaces/event-log/evtx-store.ts @@ -14,6 +14,14 @@ import { EVTX_TIME_WINDOW_MS } from "./types"; // Re-exported so callers have one import site; the implementations live in a Tauri-free module. export { parseEventIdFilter, selectVisibleRecords } from "./evtx-filter"; import type { EvtxGroupField } from "./evtx-filter"; +import { + defaultColumnConfig, + moveColumn, + sanitizeColumnConfig, + toggleColumn, + type EvtxColumnConfig, + type EvtxColumnId, +} from "./evtx-columns"; /** * Builds the filter handed to the backend, which compiles it to XPath. @@ -49,6 +57,7 @@ interface EvtxState { filterEventIds: string; filterSearch: string; timeWindow: EvtxTimeWindow; + columnConfig: EvtxColumnConfig; groupBy: EvtxGroupField[]; collapsedGroups: Set; sortField: EvtxSortField; @@ -70,6 +79,9 @@ interface EvtxState { setFilterSearch: (search: string) => void; setTimeWindow: (window: EvtxTimeWindow) => void; setGroupBy: (fields: EvtxGroupField[]) => void; + toggleColumnVisible: (id: EvtxColumnId) => void; + moveColumnBy: (id: EvtxColumnId, direction: -1 | 1) => void; + resetColumns: () => void; toggleGroup: (key: string) => void; setSortField: (field: EvtxSortField) => void; setSortDirection: (direction: EvtxSortDirection) => void; @@ -109,6 +121,7 @@ export const useEvtxStore = create()((set, get) => ({ filterEventIds: "", filterSearch: "", timeWindow: "24h", + columnConfig: defaultColumnConfig(), groupBy: [], collapsedGroups: new Set(), sortField: "time", @@ -358,6 +371,11 @@ export const useEvtxStore = create()((set, get) => ({ // Changing the grouping invalidates every collapse key, so the old set is discarded rather than // left to collapse unrelated groups that happen to share a key. setGroupBy: (fields) => set({ groupBy: fields, collapsedGroups: new Set() }), + toggleColumnVisible: (id) => + set({ columnConfig: sanitizeColumnConfig(toggleColumn(get().columnConfig, id)) }), + moveColumnBy: (id, direction) => + set({ columnConfig: sanitizeColumnConfig(moveColumn(get().columnConfig, id, direction)) }), + resetColumns: () => set({ columnConfig: defaultColumnConfig() }), toggleGroup: (key) => { const next = new Set(get().collapsedGroups); if (next.has(key)) next.delete(key); @@ -385,6 +403,7 @@ export const useEvtxStore = create()((set, get) => ({ filterEventIds: "", filterSearch: "", timeWindow: "24h", + columnConfig: defaultColumnConfig(), groupBy: [], collapsedGroups: new Set(), sortField: "time", From d3dec44480b0546b2bc43ee465f3a0263e0f175d Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 9 Aug 2026 17:42:57 -0400 Subject: [PATCH 29/85] feat(parser): merge Windows events and text logs into one timeline Phase 4, and the reason this work beats the incumbents rather than matching them. Every event viewer surveyed for #539 is events-only and every log viewer is text-only. Nobody puts them side by side, which is the one thing that actually explains a failure: the DeviceManagement provider event says enrollment failed with an HRESULT, and the IntuneManagementExtension.log line thirty seconds earlier says why. The module is pure and holds no knowledge of where items came from, so the event side can live in the host layer while the log side converts straight from LogEntry. Severity is normalized across both, since a merged view needs one ranking. The hard part is not sorting. It is refusing to place what cannot honestly be placed. An entry with no timestamp, which is every continuation line and every header, has no position; putting it at the epoch or at the previous entry's time would invent a sequence the evidence does not support. Those are returned as unplaced items carrying their file, line, and component so an operator can go and look, rather than being dropped into a timeline that then looks complete. Ordering is a stable sort. A log line and an event in the same millisecond have no discoverable ordering between them, so re-sorting on severity or source would manufacture one. Two mappings are deliberate. Log Success becomes Info rather than gaining a rank, which would sort it away from the events it sits between. Event level 0 means "not set" and becomes Info, because ranking unclassified events critical would flood a severity filter. LogEntry, Severity, and LogFormat gain Default. An 88-field struct that cannot be constructed in a test forces every test to spell out every field or skip the type entirely. LogFormat defaults to Plain because claiming a structured format that was never detected asserts more than the evidence supports. Gates: 2,304 parser tests, 1,137 src-tauri tests, clippy -D warnings clean on both crates, rustfmt clean, wasm32 builds. Refs #539. Co-Authored-By: Claude Opus 5 --- crates/cmtraceopen-parser/src/lib.rs | 1 + .../src/models/log_entry.rs | 18 +- .../src/unified_timeline/mod.rs | 392 ++++++++++++++++++ 3 files changed, 407 insertions(+), 4 deletions(-) create mode 100644 crates/cmtraceopen-parser/src/unified_timeline/mod.rs diff --git a/crates/cmtraceopen-parser/src/lib.rs b/crates/cmtraceopen-parser/src/lib.rs index 775216cea..15702390a 100644 --- a/crates/cmtraceopen-parser/src/lib.rs +++ b/crates/cmtraceopen-parser/src/lib.rs @@ -18,4 +18,5 @@ pub mod models; pub mod parser; pub mod provider; pub mod sccm; +pub mod unified_timeline; pub(crate) mod wire; diff --git a/crates/cmtraceopen-parser/src/models/log_entry.rs b/crates/cmtraceopen-parser/src/models/log_entry.rs index b7eeb515e..37db58ca6 100644 --- a/crates/cmtraceopen-parser/src/models/log_entry.rs +++ b/crates/cmtraceopen-parser/src/models/log_entry.rs @@ -4,9 +4,12 @@ use serde::{Deserialize, Serialize}; /// Maps directly to CMTrace's type field: 0=Success, 1=Info, 2=Warning, 3=Error. /// `Success` corresponds to CCM/PSADT `type="0"` — a completed operation that /// OneTrace renders with a green tick. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] pub enum Severity { Success, + /// Default: an unclassified line is informational, which is how every parser already treats + /// one it cannot rank. + #[default] Info, Warning, Error, @@ -24,13 +27,17 @@ pub enum EntryKind { } /// Which log format was detected/used to parse this entry. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] pub enum LogFormat { /// CCM/SCCM format: Ccm, /// Simple/legacy format: message$$ Simple, - /// Plain text (no structured format detected) + /// Plain text (no structured format detected). + /// + /// Default, because claiming a structured format that was never detected would be a stronger + /// assertion than the evidence supports. + #[default] Plain, /// Generic timestamped format (ISO 8601, slash-dates, syslog, time-only) Timestamped, @@ -154,7 +161,10 @@ pub struct ParserSelectionInfo { /// A single parsed log entry. /// Field names use camelCase for direct JSON serialization to TypeScript. -#[derive(Debug, Clone, Serialize, Deserialize)] +/// +/// `Default` exists so tests can construct one by naming only the fields under test; the struct +/// carries far too many fields to spell out every time. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct LogEntry { /// Sequential ID for stable row identity diff --git a/crates/cmtraceopen-parser/src/unified_timeline/mod.rs b/crates/cmtraceopen-parser/src/unified_timeline/mod.rs new file mode 100644 index 000000000..0e7246cb2 --- /dev/null +++ b/crates/cmtraceopen-parser/src/unified_timeline/mod.rs @@ -0,0 +1,392 @@ +//! Merging Windows events and parsed text logs into one chronological view. +//! +//! Every event viewer surveyed for issue #539 is events-only, and every log viewer is text-only. +//! Nobody puts them side by side, which is the one thing that actually explains a failure: the +//! `DeviceManagement-Enterprise-Diagnostics-Provider` event says enrollment failed with an HRESULT, +//! and the `IntuneManagementExtension.log` line thirty seconds earlier says why. +//! +//! This module owns the merge. It is pure and holds no knowledge of where items came from, so the +//! event side can live in the host layer while the log side comes straight from +//! [`LogEntry`](crate::models::log_entry::LogEntry). +//! +//! The hard part is not sorting. It is refusing to place things that cannot honestly be placed: +//! an entry with no timestamp has no position on a timeline, and putting it at the epoch or at the +//! previous entry's time would invent a sequence the evidence does not support. + +use serde::{Deserialize, Serialize}; + +use crate::models::log_entry::{LogEntry, Severity}; + +/// Severity normalized across both sides of the merge. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum TimelineSeverity { + /// Diagnostic detail, from a verbose event. Text logs have no equivalent. + Verbose, + #[default] + Info, + Warning, + Error, + /// Only events carry this; text logs top out at error. + Critical, +} + +impl TimelineSeverity { + /// Maps a text log severity. + /// + /// `Success` maps to `Info` rather than gaining a level of its own: on a merged timeline it is + /// an ordinary informational line, and a separate rank would sort it away from the events it + /// sits between. + pub fn from_log(severity: Severity) -> Self { + match severity { + Severity::Error => Self::Error, + Severity::Warning => Self::Warning, + Severity::Info | Severity::Success => Self::Info, + } + } + + /// Maps a Windows event level value as written in `System/Level`. + /// + /// Level 0 means "not set", which providers use for events that are not classified. It maps to + /// `Info` because treating an unclassified event as critical would flood a severity filter. + pub fn from_event_level(level: u8) -> Self { + match level { + 1 => Self::Critical, + 2 => Self::Error, + 3 => Self::Warning, + 5 => Self::Verbose, + _ => Self::Info, + } + } +} + +/// Where a timeline item came from. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "camelCase")] +pub enum TimelineOrigin { + /// A line from a parsed text log. + Log { + /// File the line came from, as the parser recorded it. + file: String, + /// Emitting component, when the format carries one. + component: Option, + /// 1-based line number, so the item can be traced back to the source. + line: u32, + }, + /// A Windows event. + Event { + channel: String, + provider: String, + event_id: u32, + /// Record identifier within its channel. + record_id: u64, + }, +} + +/// One placed item. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TimelineItem { + /// Milliseconds since the Unix epoch. + pub timestamp_ms: i64, + pub severity: TimelineSeverity, + pub message: String, + pub origin: TimelineOrigin, +} + +/// Something that could not be placed, and why. +/// +/// Surfaced rather than dropped: a timeline that silently omits a third of a log file looks +/// complete and is not. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct UnplacedItem { + /// Where it came from, so an operator can go and look. + pub origin: TimelineOrigin, + /// Why it has no position. + pub reason: UnplacedReason, +} + +/// Why an item has no position on the timeline. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum UnplacedReason { + /// The source carried no timestamp, or one the parser could not read. + /// + /// Common in continuation lines and in text logs whose first line is a header. + MissingTimestamp, +} + +/// A merged timeline plus everything that could not be placed on it. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct UnifiedTimeline { + /// Items in chronological order. + pub items: Vec, + /// Items with no honest position, in input order. + pub unplaced: Vec, +} + +impl UnifiedTimeline { + /// True when everything supplied was placed. + pub fn is_complete(&self) -> bool { + self.unplaced.is_empty() + } + + /// Inclusive time span covered, or `None` when nothing was placed. + pub fn span_ms(&self) -> Option<(i64, i64)> { + let first = self.items.first()?.timestamp_ms; + let last = self.items.last()?.timestamp_ms; + Some((first, last)) + } +} + +/// Converts a parsed log entry, or reports why it cannot be placed. +pub fn from_log_entry(entry: &LogEntry) -> Result { + let origin = TimelineOrigin::Log { + file: entry + .source_file + .clone() + .unwrap_or_else(|| entry.file_path.clone()), + component: entry.component.clone(), + line: entry.line_number, + }; + + match entry.timestamp { + Some(timestamp_ms) => Ok(TimelineItem { + timestamp_ms, + severity: TimelineSeverity::from_log(entry.severity), + message: entry.message.clone(), + origin, + }), + None => Err(UnplacedItem { + origin, + reason: UnplacedReason::MissingTimestamp, + }), + } +} + +/// Merges already-converted items into one chronological timeline. +/// +/// Ordering is stable: items sharing a timestamp keep the order they were supplied in. That matters +/// because a text log and an event recorded in the same millisecond have no discoverable ordering +/// between them, and re-sorting on severity or source would invent one. +pub fn merge( + placed: impl IntoIterator, + unplaced: impl IntoIterator, +) -> UnifiedTimeline { + let mut items: Vec = placed.into_iter().collect(); + // sort_by_key is a stable sort, which is what preserves input order within a timestamp. + items.sort_by_key(|item| item.timestamp_ms); + UnifiedTimeline { + items, + unplaced: unplaced.into_iter().collect(), + } +} + +/// Converts and merges a slice of log entries, collecting the ones that cannot be placed. +pub fn from_log_entries(entries: &[LogEntry]) -> UnifiedTimeline { + let mut placed = Vec::new(); + let mut unplaced = Vec::new(); + for entry in entries { + match from_log_entry(entry) { + Ok(item) => placed.push(item), + Err(reason) => unplaced.push(reason), + } + } + merge(placed, unplaced) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn log_entry(timestamp: Option, message: &str, severity: Severity) -> LogEntry { + LogEntry { + id: 0, + line_number: 7, + message: message.to_string(), + component: Some("IME".to_string()), + timestamp, + severity, + file_path: "C:/logs/IntuneManagementExtension.log".to_string(), + ..LogEntry::default() + } + } + + fn event(timestamp_ms: i64, message: &str, severity: TimelineSeverity) -> TimelineItem { + TimelineItem { + timestamp_ms, + severity, + message: message.to_string(), + origin: TimelineOrigin::Event { + channel: "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider/Admin" + .to_string(), + provider: "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider" + .to_string(), + event_id: 76, + record_id: 1, + }, + } + } + + #[test] + fn events_and_log_lines_interleave_by_time() { + // The whole point: the event says enrollment failed, the log line before it says why. + let logs = [ + log_entry(Some(1_000), "Checking enrollment", Severity::Info), + log_entry(Some(3_000), "Token request rejected", Severity::Error), + ]; + let timeline = merge( + logs.iter() + .filter_map(|entry| from_log_entry(entry).ok()) + .chain([event(2_000, "MDM enroll failed", TimelineSeverity::Error)]), + [], + ); + + let messages: Vec<&str> = timeline.items.iter().map(|i| i.message.as_str()).collect(); + assert_eq!( + messages, + vec![ + "Checking enrollment", + "MDM enroll failed", + "Token request rejected" + ] + ); + } + + #[test] + fn an_entry_without_a_timestamp_is_reported_rather_than_placed() { + // Placing it at the epoch, or at the previous entry's time, would invent a sequence. + let entries = [ + log_entry(Some(1_000), "placed", Severity::Info), + log_entry(None, "continuation line", Severity::Info), + ]; + let timeline = from_log_entries(&entries); + + assert_eq!(timeline.items.len(), 1); + assert_eq!(timeline.unplaced.len(), 1); + assert_eq!( + timeline.unplaced[0].reason, + UnplacedReason::MissingTimestamp + ); + assert!(!timeline.is_complete()); + } + + #[test] + fn an_unplaced_item_still_says_where_it_came_from() { + let timeline = from_log_entries(&[log_entry(None, "orphan", Severity::Error)]); + match &timeline.unplaced[0].origin { + TimelineOrigin::Log { + file, + line, + component, + } => { + assert!(file.ends_with("IntuneManagementExtension.log")); + assert_eq!(*line, 7); + assert_eq!(component.as_deref(), Some("IME")); + } + other => panic!("expected a log origin, got {other:?}"), + } + } + + #[test] + fn items_sharing_a_timestamp_keep_their_input_order() { + // A log line and an event in the same millisecond have no discoverable ordering, so the + // merge must not invent one by sorting on severity or source. + let timeline = merge( + [ + event(5_000, "first supplied", TimelineSeverity::Error), + event(5_000, "second supplied", TimelineSeverity::Verbose), + event(5_000, "third supplied", TimelineSeverity::Critical), + ], + [], + ); + let messages: Vec<&str> = timeline.items.iter().map(|i| i.message.as_str()).collect(); + assert_eq!( + messages, + vec!["first supplied", "second supplied", "third supplied"] + ); + } + + #[test] + fn negative_timestamps_sort_before_the_epoch_rather_than_wrapping() { + let timeline = merge( + [ + event(10, "after", TimelineSeverity::Info), + event(-10, "before", TimelineSeverity::Info), + ], + [], + ); + assert_eq!(timeline.items[0].message, "before"); + } + + #[test] + fn the_span_covers_first_to_last() { + let timeline = merge( + [ + event(3_000, "c", TimelineSeverity::Info), + event(1_000, "a", TimelineSeverity::Info), + ], + [], + ); + assert_eq!(timeline.span_ms(), Some((1_000, 3_000))); + } + + #[test] + fn an_empty_timeline_has_no_span_rather_than_a_zero_one() { + let timeline = UnifiedTimeline::default(); + assert_eq!(timeline.span_ms(), None); + assert!(timeline.is_complete()); + } + + #[test] + fn log_success_is_informational_not_a_rank_of_its_own() { + // A separate rank would sort Success away from the events it sits between. + assert_eq!( + TimelineSeverity::from_log(Severity::Success), + TimelineSeverity::Info + ); + assert_eq!( + TimelineSeverity::from_log(Severity::Error), + TimelineSeverity::Error + ); + } + + #[test] + fn event_level_zero_is_informational_not_critical() { + // Level 0 means "not set". Ranking it critical would flood a severity filter. + assert_eq!( + TimelineSeverity::from_event_level(0), + TimelineSeverity::Info + ); + assert_eq!( + TimelineSeverity::from_event_level(1), + TimelineSeverity::Critical + ); + assert_eq!( + TimelineSeverity::from_event_level(5), + TimelineSeverity::Verbose + ); + // An undeclared level must not panic or invent a rank. + assert_eq!( + TimelineSeverity::from_event_level(9), + TimelineSeverity::Info + ); + } + + #[test] + fn severity_orders_from_verbose_to_critical() { + assert!(TimelineSeverity::Verbose < TimelineSeverity::Info); + assert!(TimelineSeverity::Error < TimelineSeverity::Critical); + } + + #[test] + fn a_log_entry_falls_back_to_its_file_path_when_no_source_file_is_recorded() { + let timeline = from_log_entries(&[log_entry(Some(1), "x", Severity::Info)]); + match &timeline.items[0].origin { + TimelineOrigin::Log { file, .. } => assert!(file.contains("IntuneManagementExtension")), + other => panic!("expected a log origin, got {other:?}"), + } + } +} From c4805ea91104838b63acc4aaaed3efb5268fe49f Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 9 Aug 2026 17:47:04 -0400 Subject: [PATCH 30/85] feat(event-log): place events on the unified timeline and expose it over IPC The event-side adapter for the merge landed in the parser crate. The log side needs no adapter because LogEntry already lives there. A record whose timestamp did not parse carries timestamp_epoch 0, which is 1970 and not a time any Windows event was written. Treating that as a real position would drop the event at the far left of every timeline and imply it happened first, so it is reported as unplaced instead, carrying its channel, event id and record id so an operator can go and look. Both sides of the merge arrive from the frontend rather than being re-read, so the timeline covers exactly what the operator has open. Re-reading would risk building a timeline from a different set than the one on screen, which is the same reason export sends its records rather than re-querying. Gates: 1,143 src-tauri tests, clippy -D warnings clean on full and lite feature sets, rustfmt clean. Refs #539. Co-Authored-By: Claude Opus 5 --- src-tauri/src/event_log/commands.rs | 21 +++ src-tauri/src/event_log/mod.rs | 1 + src-tauri/src/event_log/timeline.rs | 211 ++++++++++++++++++++++++++++ src-tauri/src/lib.rs | 2 + 4 files changed, 235 insertions(+) create mode 100644 src-tauri/src/event_log/timeline.rs diff --git a/src-tauri/src/event_log/commands.rs b/src-tauri/src/event_log/commands.rs index 7366a9231..a9832e8b6 100644 --- a/src-tauri/src/event_log/commands.rs +++ b/src-tauri/src/event_log/commands.rs @@ -207,3 +207,24 @@ pub async fn evtx_load_provider_databases( pub async fn evtx_provider_databases() -> Vec { super::provider_db::registered() } + +/// Merges already-loaded log entries and event records into one chronological timeline. +/// +/// Both sides arrive from the frontend rather than being re-read, so the timeline covers exactly +/// what the operator has open. Re-reading would risk building a timeline from a different set than +/// the one on screen. +#[tauri::command] +pub async fn evtx_build_unified_timeline( + entries: Vec, + records: Vec, +) -> Result { + let timeline = tokio::task::spawn_blocking(move || super::timeline::build(&entries, &records)) + .await + .map_err(|error| format!("timeline build task failed: {error}"))?; + log::info!( + "event=unified_timeline items={} unplaced={}", + timeline.items.len(), + timeline.unplaced.len() + ); + Ok(timeline) +} diff --git a/src-tauri/src/event_log/mod.rs b/src-tauri/src/event_log/mod.rs index 5e9927c84..f7063a0aa 100644 --- a/src-tauri/src/event_log/mod.rs +++ b/src-tauri/src/event_log/mod.rs @@ -5,6 +5,7 @@ pub mod maps; pub mod models; pub mod parser; pub mod provider_db; +pub mod timeline; #[cfg(target_os = "windows")] pub mod live; diff --git a/src-tauri/src/event_log/timeline.rs b/src-tauri/src/event_log/timeline.rs new file mode 100644 index 000000000..203806a0e --- /dev/null +++ b/src-tauri/src/event_log/timeline.rs @@ -0,0 +1,211 @@ +//! Placing Windows events onto the unified timeline. +//! +//! The merge itself lives in `cmtraceopen_parser::unified_timeline`, which is pure and knows +//! nothing about where items came from. This is the event-side adapter: it converts an +//! [`EvtxRecord`] into a timeline item, or reports why it cannot be placed. +//! +//! The log side needs no adapter because `LogEntry` already lives in the parser crate. + +use cmtraceopen_parser::models::log_entry::LogEntry; +use cmtraceopen_parser::unified_timeline::{ + from_log_entry, merge, TimelineItem, TimelineOrigin, TimelineSeverity, UnifiedTimeline, + UnplacedItem, UnplacedReason, +}; + +use super::models::{EvtxLevel, EvtxRecord}; + +/// Maps the record's level back to a timeline severity. +/// +/// `EvtxRecord` stores a decoded level rather than the raw `System/Level` value, so this maps the +/// decoded form. Information is the resting state, matching how the decoder treats a level it does +/// not recognise. +fn severity_of(level: EvtxLevel) -> TimelineSeverity { + match level { + EvtxLevel::Critical => TimelineSeverity::Critical, + EvtxLevel::Error => TimelineSeverity::Error, + EvtxLevel::Warning => TimelineSeverity::Warning, + EvtxLevel::Verbose => TimelineSeverity::Verbose, + EvtxLevel::Information => TimelineSeverity::Info, + } +} + +fn origin_of(record: &EvtxRecord) -> TimelineOrigin { + TimelineOrigin::Event { + channel: record.channel.clone(), + provider: record.provider.clone(), + event_id: record.event_id, + record_id: record.event_record_id, + } +} + +/// Converts one event, or reports why it has no position. +/// +/// A record whose timestamp did not parse carries `timestamp_epoch == 0`, which is 1970 and not a +/// time any Windows event was written. Treating it as a real position would drop the event at the +/// far left of every timeline and imply it happened first. +pub fn from_event(record: &EvtxRecord) -> Result { + if record.timestamp_epoch == 0 { + return Err(UnplacedItem { + origin: origin_of(record), + reason: UnplacedReason::MissingTimestamp, + }); + } + + Ok(TimelineItem { + timestamp_ms: record.timestamp_epoch, + severity: severity_of(record.level), + message: record.message.clone(), + origin: origin_of(record), + }) +} + +/// Builds one timeline from parsed log entries and events. +pub fn build(entries: &[LogEntry], records: &[EvtxRecord]) -> UnifiedTimeline { + let mut placed = Vec::with_capacity(entries.len() + records.len()); + let mut unplaced = Vec::new(); + + for entry in entries { + match from_log_entry(entry) { + Ok(item) => placed.push(item), + Err(reason) => unplaced.push(reason), + } + } + for record in records { + match from_event(record) { + Ok(item) => placed.push(item), + Err(reason) => unplaced.push(reason), + } + } + + merge(placed, unplaced) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn record(timestamp_epoch: i64, message: &str, level: EvtxLevel) -> EvtxRecord { + EvtxRecord { + id: 0, + event_record_id: 76, + timestamp: String::new(), + timestamp_epoch, + provider: "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider" + .to_string(), + channel: "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider/Admin" + .to_string(), + event_id: 76, + level, + computer: "RING0IVY24-01".to_string(), + message: message.to_string(), + event_data: Vec::new(), + raw_xml: String::new(), + source_label: "Live".to_string(), + task: None, + opcode: None, + process_id: None, + thread_id: None, + user_sid: None, + keywords: None, + mapped: Vec::new(), + } + } + + fn entry(timestamp: Option, message: &str) -> LogEntry { + LogEntry { + line_number: 12, + message: message.to_string(), + component: Some("IME".to_string()), + timestamp, + file_path: "C:/logs/IntuneManagementExtension.log".to_string(), + ..LogEntry::default() + } + } + + #[test] + fn an_event_and_the_log_lines_around_it_interleave() { + // The case the whole feature exists for. + let timeline = build( + &[ + entry(Some(1_000), "Checking enrollment"), + entry(Some(3_000), "Token request rejected"), + ], + &[record(2_000, "MDM enroll failed", EvtxLevel::Error)], + ); + + let messages: Vec<&str> = timeline.items.iter().map(|i| i.message.as_str()).collect(); + assert_eq!( + messages, + vec![ + "Checking enrollment", + "MDM enroll failed", + "Token request rejected" + ] + ); + assert!(timeline.is_complete()); + } + + #[test] + fn an_event_with_no_parsed_timestamp_is_unplaced_rather_than_dated_to_1970() { + // Epoch zero would put it at the far left of every timeline and imply it happened first. + let timeline = build(&[], &[record(0, "undated", EvtxLevel::Error)]); + assert!(timeline.items.is_empty()); + assert_eq!(timeline.unplaced.len(), 1); + assert_eq!( + timeline.unplaced[0].reason, + UnplacedReason::MissingTimestamp + ); + } + + #[test] + fn an_unplaced_event_still_identifies_itself() { + let timeline = build(&[], &[record(0, "undated", EvtxLevel::Error)]); + match &timeline.unplaced[0].origin { + TimelineOrigin::Event { + channel, + event_id, + record_id, + .. + } => { + assert!(channel.contains("DeviceManagement")); + assert_eq!(*event_id, 76); + assert_eq!(*record_id, 76); + } + other => panic!("expected an event origin, got {other:?}"), + } + } + + #[test] + fn unplaced_items_from_both_sides_are_collected_together() { + let timeline = build( + &[entry(None, "continuation line")], + &[record(0, "undated event", EvtxLevel::Information)], + ); + assert_eq!(timeline.unplaced.len(), 2); + assert!(!timeline.is_complete()); + } + + #[test] + fn every_level_maps_without_panicking() { + for level in [ + EvtxLevel::Critical, + EvtxLevel::Error, + EvtxLevel::Warning, + EvtxLevel::Information, + EvtxLevel::Verbose, + ] { + let item = from_event(&record(1, "x", level)).expect("placed"); + assert_eq!(item.timestamp_ms, 1); + } + assert_eq!(severity_of(EvtxLevel::Critical), TimelineSeverity::Critical); + assert_eq!(severity_of(EvtxLevel::Verbose), TimelineSeverity::Verbose); + } + + #[test] + fn an_empty_build_is_complete_and_empty() { + let timeline = build(&[], &[]); + assert!(timeline.items.is_empty()); + assert!(timeline.is_complete()); + assert_eq!(timeline.span_ms(), None); + } +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 6bf20949f..111c2a638 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -363,6 +363,8 @@ pub fn run() { event_log::commands::evtx_load_provider_databases, #[cfg(feature = "event-log")] event_log::commands::evtx_provider_databases, + #[cfg(feature = "event-log")] + event_log::commands::evtx_build_unified_timeline, #[cfg(target_os = "windows")] commands::graph_api::graph_authenticate, #[cfg(target_os = "windows")] From 3309070dcc21acb5c2db898ad679ffd0d1ae9faf Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 9 Aug 2026 17:48:08 -0400 Subject: [PATCH 31/85] feat(event-log): frontend model for the unified timeline Mirrors the Rust type. The merge happens in Rust; this side handles presentation, which mostly means deciding what to say about items that could not be placed. Source labels are shortened for a column narrow enough to scan. A log shows its file name and component rather than the full path, and an event shows the channel leaf rather than the Microsoft-Windows prefix that sits on nearly every channel and distinguishes nothing. The full value is available as a tooltip. The unplaced summary returns null rather than "0 items" when nothing was dropped, so the caller hides the notice entirely. A reassuring zero invites no attention, and the whole point of surfacing unplaced items is that they should be noticed when they exist. It also names only the source that actually contributed, so an operator reading "1 event could not be placed" is not left wondering about log lines that were all fine. Gates: 70 frontend tests, tsc clean. Refs #539. Co-Authored-By: Claude Opus 5 --- .../event-log/unified-timeline.test.ts | 124 ++++++++++++++++++ src/workspaces/event-log/unified-timeline.ts | 110 ++++++++++++++++ 2 files changed, 234 insertions(+) create mode 100644 src/workspaces/event-log/unified-timeline.test.ts create mode 100644 src/workspaces/event-log/unified-timeline.ts diff --git a/src/workspaces/event-log/unified-timeline.test.ts b/src/workspaces/event-log/unified-timeline.test.ts new file mode 100644 index 000000000..618de93ca --- /dev/null +++ b/src/workspaces/event-log/unified-timeline.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it } from "vitest"; +import { + isEventOrigin, + originDetail, + originLabel, + timelineCounts, + unplacedSummary, + type TimelineOrigin, + type UnifiedTimeline, +} from "./unified-timeline"; + +const logOrigin: TimelineOrigin = { + kind: "log", + file: "C:\\ProgramData\\Microsoft\\IntuneManagementExtension\\Logs\\IntuneManagementExtension.log", + component: "IME", + line: 42, +}; + +const eventOrigin: TimelineOrigin = { + kind: "event", + channel: "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider/Admin", + provider: "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider", + eventId: 76, + recordId: 1234, +}; + +function timeline(partial: Partial = {}): UnifiedTimeline { + return { items: [], unplaced: [], ...partial }; +} + +describe("originLabel", () => { + it("shows the log file name and component, not the whole path", () => { + expect(originLabel(logOrigin)).toBe("IntuneManagementExtension.log [IME]"); + }); + + it("omits the component when the format has none", () => { + expect(originLabel({ ...logOrigin, component: null })).toBe( + "IntuneManagementExtension.log" + ); + }); + + it("shows the channel leaf and event id, not the Microsoft-Windows prefix", () => { + // That prefix is on nearly every channel and distinguishes nothing in a narrow column. + expect(originLabel(eventOrigin)).toBe("Admin (76)"); + }); + + it("falls back to the whole value when there is no separator", () => { + expect(originLabel({ ...eventOrigin, channel: "Security" })).toBe("Security (76)"); + expect(originLabel({ ...logOrigin, file: "app.log", component: null })).toBe("app.log"); + }); +}); + +describe("originDetail", () => { + it("gives the full path and line for a log", () => { + expect(originDetail(logOrigin)).toContain("IntuneManagementExtension.log:42"); + expect(originDetail(logOrigin)).toContain("(IME)"); + }); + + it("gives channel, provider, event and record for an event", () => { + const detail = originDetail(eventOrigin); + expect(detail).toContain("event 76"); + expect(detail).toContain("record 1234"); + }); +}); + +describe("isEventOrigin", () => { + it("distinguishes the two sources", () => { + expect(isEventOrigin(eventOrigin)).toBe(true); + expect(isEventOrigin(logOrigin)).toBe(false); + }); +}); + +describe("unplacedSummary", () => { + it("returns null when nothing was dropped", () => { + // A reassuring "0 items" invites no attention, so the caller hides the notice entirely. + expect(unplacedSummary(timeline())).toBeNull(); + }); + + it("counts both sources and says why", () => { + const summary = unplacedSummary( + timeline({ + unplaced: [ + { origin: logOrigin, reason: "missingTimestamp" }, + { origin: logOrigin, reason: "missingTimestamp" }, + { origin: eventOrigin, reason: "missingTimestamp" }, + ], + }) + ); + expect(summary).toBe("2 log lines and 1 event could not be placed: no timestamp"); + }); + + it("uses singular wording for a single item", () => { + expect( + unplacedSummary(timeline({ unplaced: [{ origin: logOrigin, reason: "missingTimestamp" }] })) + ).toBe("1 log line could not be placed: no timestamp"); + }); + + it("mentions only the source that actually contributed", () => { + const summary = unplacedSummary( + timeline({ unplaced: [{ origin: eventOrigin, reason: "missingTimestamp" }] }) + ); + expect(summary).toBe("1 event could not be placed: no timestamp"); + }); +}); + +describe("timelineCounts", () => { + it("separates events from log lines", () => { + const counts = timelineCounts( + timeline({ + items: [ + { timestampMs: 1, severity: "info", message: "a", origin: logOrigin }, + { timestampMs: 2, severity: "error", message: "b", origin: eventOrigin }, + { timestampMs: 3, severity: "info", message: "c", origin: logOrigin }, + ], + unplaced: [{ origin: logOrigin, reason: "missingTimestamp" }], + }) + ); + expect(counts).toEqual({ logs: 2, events: 1, unplaced: 1 }); + }); + + it("counts an empty timeline as zero everywhere", () => { + expect(timelineCounts(timeline())).toEqual({ logs: 0, events: 0, unplaced: 0 }); + }); +}); diff --git a/src/workspaces/event-log/unified-timeline.ts b/src/workspaces/event-log/unified-timeline.ts new file mode 100644 index 000000000..d479c5881 --- /dev/null +++ b/src/workspaces/event-log/unified-timeline.ts @@ -0,0 +1,110 @@ +/** + * Frontend model for the unified timeline. + * + * Mirrors `cmtraceopen_parser::unified_timeline`. The merge itself happens in Rust; this side + * handles presentation, which mostly means deciding what to show about items that could not be + * placed. + */ + +export type TimelineSeverity = + | "verbose" + | "info" + | "warning" + | "error" + | "critical"; + +export type TimelineOrigin = + | { kind: "log"; file: string; component: string | null; line: number } + | { + kind: "event"; + channel: string; + provider: string; + eventId: number; + recordId: number; + }; + +export interface TimelineItem { + timestampMs: number; + severity: TimelineSeverity; + message: string; + origin: TimelineOrigin; +} + +export interface UnplacedItem { + origin: TimelineOrigin; + reason: "missingTimestamp"; +} + +export interface UnifiedTimeline { + items: TimelineItem[]; + unplaced: UnplacedItem[]; +} + +export const TIMELINE_SEVERITY_RANK: Record = { + verbose: 0, + info: 1, + warning: 2, + error: 3, + critical: 4, +}; + +/** Short label for an item's source, for a column narrow enough to scan. */ +export function originLabel(origin: TimelineOrigin): string { + if (origin.kind === "log") { + // The file name alone; the full path is available as a tooltip and is far too long to scan. + const name = origin.file.split(/[\\/]/).pop() || origin.file; + return origin.component ? `${name} [${origin.component}]` : name; + } + // The leaf of the channel path, since the Microsoft-Windows- prefix is on nearly every channel + // and carries no distinguishing information in a narrow column. + const leaf = origin.channel.split("/").pop() || origin.channel; + return `${leaf} (${origin.eventId})`; +} + +/** Full source description, for a tooltip. */ +export function originDetail(origin: TimelineOrigin): string { + if (origin.kind === "log") { + return `${origin.file}:${origin.line}${origin.component ? ` (${origin.component})` : ""}`; + } + return `${origin.channel} / ${origin.provider} / event ${origin.eventId} / record ${origin.recordId}`; +} + +/** True when the item came from a Windows event rather than a text log. */ +export function isEventOrigin(origin: TimelineOrigin): boolean { + return origin.kind === "event"; +} + +/** + * Human-readable summary of what could not be placed. + * + * Returns null when nothing was dropped, so the caller can hide the notice entirely rather than + * showing a reassuring "0 items" that invites no attention. + */ +export function unplacedSummary(timeline: UnifiedTimeline): string | null { + const total = timeline.unplaced.length; + if (total === 0) return null; + + const logs = timeline.unplaced.filter((item) => item.origin.kind === "log").length; + const events = total - logs; + + const parts: string[] = []; + if (logs > 0) parts.push(`${logs} log ${logs === 1 ? "line" : "lines"}`); + if (events > 0) parts.push(`${events} ${events === 1 ? "event" : "events"}`); + + return `${parts.join(" and ")} could not be placed: no timestamp`; +} + +/** Counts by source kind, for a coverage readout. */ +export function timelineCounts(timeline: UnifiedTimeline): { + logs: number; + events: number; + unplaced: number; +} { + let logs = 0; + let events = 0; + for (const item of timeline.items) { + if (item.origin.kind === "event") events += 1; + else logs += 1; + } + return { logs, events, unplaced: timeline.unplaced.length }; +} From 8de9b26004c18cc81b464d2546a7f1232ab4cd8c Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 9 Aug 2026 17:49:39 -0400 Subject: [PATCH 32/85] feat(event-log): render the unified timeline Completes Phase 4 through the UI. Events and text log lines interleave in one virtualized list, each row carrying a source badge because knowing at a glance which side a line came from is the whole value of the view. Without it the two blur together and the reader loses the distinction that makes the correlation mean anything. Source labels are shortened to stay scannable and carry the full value as a tooltip: a log shows its file name and component, an event shows the channel leaf and id rather than the Microsoft-Windows prefix that sits on nearly every channel. The header reports log and event counts, and mentions unplaced items only when there are some. A "0 unplaced" badge would read as reassurance and invite no attention, which defeats the reason for surfacing them at all. Its tooltip says why they were dropped rather than leaving the operator to guess. Gates: 70 frontend tests, tsc clean, vitest exits 0, clippy clean. Refs #539. Co-Authored-By: Claude Opus 5 --- .../event-log/UnifiedTimelineView.tsx | 218 ++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 src/workspaces/event-log/UnifiedTimelineView.tsx diff --git a/src/workspaces/event-log/UnifiedTimelineView.tsx b/src/workspaces/event-log/UnifiedTimelineView.tsx new file mode 100644 index 000000000..70fb38d40 --- /dev/null +++ b/src/workspaces/event-log/UnifiedTimelineView.tsx @@ -0,0 +1,218 @@ +import { useMemo, useRef } from "react"; +import { tokens } from "@fluentui/react-components"; +import { useVirtualizer } from "@tanstack/react-virtual"; + +import { LOG_MONOSPACE_FONT_FAMILY, LOG_UI_FONT_FAMILY, getLogListMetrics } from "../../lib/log-accessibility"; +import { useUiStore } from "../../stores/ui-store"; +import { + isEventOrigin, + originDetail, + originLabel, + timelineCounts, + unplacedSummary, + type TimelineSeverity, + type UnifiedTimeline, +} from "./unified-timeline"; + +const SEVERITY_COLORS: Record = { + critical: tokens.colorPaletteRedForeground1, + error: tokens.colorPaletteRedForeground1, + warning: tokens.colorPaletteMarigoldForeground1, + info: tokens.colorBrandForeground1, + verbose: tokens.colorNeutralForeground4, +}; + +function formatTimestamp(ms: number): string { + const date = new Date(ms); + const pad = (value: number, width = 2) => String(value).padStart(width, "0"); + return ( + `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())} ` + + `${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}.` + + `${pad(date.getMilliseconds(), 3)}` + ); +} + +export interface UnifiedTimelineViewProps { + timeline: UnifiedTimeline; +} + +/** + * The merged view of events and text logs. + * + * Rows carry a source badge because the whole value of the view is knowing, at a glance, which + * side of the merge a line came from. Without it the two blur together and the reader loses the + * distinction that makes the correlation meaningful. + */ +export function UnifiedTimelineView({ timeline }: UnifiedTimelineViewProps) { + const logListFontSize = useUiStore((s) => s.logListFontSize); + const metrics = useMemo(() => getLogListMetrics(logListFontSize), [logListFontSize]); + + const parentRef = useRef(null); + const virtualizer = useVirtualizer({ + count: timeline.items.length, + getScrollElement: () => parentRef.current, + estimateSize: () => metrics.rowHeight + 2, + overscan: 12, + }); + + const counts = useMemo(() => timelineCounts(timeline), [timeline]); + const dropped = useMemo(() => unplacedSummary(timeline), [timeline]); + + const fontSize = metrics.fontSize; + const smallFontSize = Math.max(9, fontSize - 3); + const monoFontSize = Math.max(10, fontSize - 1); + + return ( +
+
+ {counts.logs.toLocaleString()} log lines + {counts.events.toLocaleString()} events + {/* Only shown when something was actually dropped; a "0 unplaced" badge would read as + reassurance and invite no attention. */} + {dropped && ( + + {dropped} + + )} +
+ + {timeline.items.length === 0 ? ( +
+ Nothing to place on the timeline yet. Load a log file and an event source to correlate + them. +
+ ) : ( +
+
+ {virtualizer.getVirtualItems().map((virtualRow) => { + const item = timeline.items[virtualRow.index]; + if (!item) return null; + const color = SEVERITY_COLORS[item.severity]; + const fromEvent = isEventOrigin(item.origin); + + return ( +
+ + {fromEvent ? "EVT" : "LOG"} + + + + {formatTimestamp(item.timestampMs)} + + + + {originLabel(item.origin)} + + + + {item.message} + +
+ ); + })} +
+
+ )} +
+ ); +} From adfeb115a55473dafa97b2195fdd0502a9d78659 Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 9 Aug 2026 18:09:11 -0400 Subject: [PATCH 33/85] feat(event-log): decode .etl payloads and fix the dead file-open path Trace-backed channels write their message as a hex blob in an EventPayload element rather than as EventData. Rendered raw it is a wall of hex digits, which is why C:\Windows\Logs\WindowsUpdate is unreadable without a dedicated tool. Decoding is by inspection rather than assumption: both UTF-16LE and UTF-8 readings are scored and the better one wins, with a refusal when neither is convincing, because mojibake looks like data and misleads where raw hex is at least visibly unreadable. Wiring it up surfaced a larger defect. The file-open path iterated records_json_value and then fed the pretty-printed JSON to parse_event_xml, which can never succeed. Every consumer of the parsed tree was therefore dead on an opened .evtx file: the System block (Task, Opcode, PID, TID, User, Keywords all empty), every registered map, and the XML export, which emitted JSON under an Events root. The path now reads XML, and identity comes off the same tree the live path uses. Field extraction moves with it. UserData is now read alongside EventData, so classic and trace-backed events have fields at all, and positional Data is numbered from one to line up with the provider's insertion template. - readable_ratio replaces a printable check that accepted Latin Extended mojibake as text - timestamp parsing is shared, so a zoneless stamp is read as UTC on both paths rather than sorting to 1970 on one - an unparsable record is counted as an error instead of pushed as provider "Unknown" at the epoch Co-Authored-By: Claude Opus 5 --- .../src/event_payload/mod.rs | 298 ++++++++++++++++++ crates/cmtraceopen-parser/src/lib.rs | 1 + src-tauri/src/event_log/event_node.rs | 22 +- src-tauri/src/event_log/live.rs | 41 +-- src-tauri/src/event_log/mod.rs | 41 ++- src-tauri/src/event_log/parser.rs | 275 ++++++++++------ 6 files changed, 562 insertions(+), 116 deletions(-) create mode 100644 crates/cmtraceopen-parser/src/event_payload/mod.rs diff --git a/crates/cmtraceopen-parser/src/event_payload/mod.rs b/crates/cmtraceopen-parser/src/event_payload/mod.rs new file mode 100644 index 000000000..fafad902d --- /dev/null +++ b/crates/cmtraceopen-parser/src/event_payload/mod.rs @@ -0,0 +1,298 @@ +//! Decoding the `EventPayload` element found in `.etl` traces. +//! +//! Some providers, notably the Windows Update trace under `C:\Windows\Logs\WindowsUpdate`, do not +//! write structured `EventData`. They write a single `EventPayload` element holding the message as +//! a hexadecimal string. Rendered as-is it is a wall of hex digits, which is why those traces are +//! usually described as unreadable without a dedicated tool. +//! +//! FullEventLogView added this conversion in 1.55 and it is the reason those logs are readable +//! there at all. This is the same idea, implemented as a pure function. +//! +//! Encoding is decided by inspection rather than assumption. Windows writes these payloads as +//! UTF-16LE, but not universally, and guessing wrong turns readable text into interleaved nulls or +//! mojibake. Both interpretations are scored and the better one wins, with a refusal when neither +//! is convincing, in which case the caller shows the hex unchanged. + +use crate::eventmap::EventNode; + +/// How the payload bytes were interpreted. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PayloadEncoding { + /// UTF-16 little endian, which is what Windows writes most often. + Utf16Le, + /// Single-byte text, either ASCII or UTF-8. + Utf8, +} + +/// A decoded payload. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DecodedPayload { + pub text: String, + pub encoding: PayloadEncoding, +} + +/// Parses a hexadecimal string into bytes. +/// +/// Whitespace is ignored, since providers sometimes wrap long payloads. An odd digit count or a +/// non-hex character means this is not a hex payload at all, which is a refusal rather than a best +/// effort: decoding half a string would present truncated evidence as complete. +fn hex_to_bytes(hex: &str) -> Option> { + let digits: Vec = hex + .bytes() + .filter(|byte| !byte.is_ascii_whitespace()) + .collect(); + if digits.is_empty() || !digits.len().is_multiple_of(2) { + return None; + } + + let mut bytes = Vec::with_capacity(digits.len() / 2); + for pair in digits.chunks_exact(2) { + let high = (pair[0] as char).to_digit(16)?; + let low = (pair[1] as char).to_digit(16)?; + bytes.push(((high << 4) | low) as u8); + } + Some(bytes) +} + +/// Fraction of characters that are ASCII printable or ordinary whitespace. +/// +/// Deliberately stricter than "not a control character". Misreading arbitrary bytes as UTF-16 +/// produces letter-like characters from the Latin Extended and CJK ranges, which are not control +/// characters and would sail through a looser check. Requiring ASCII rejects that, because the +/// signature of a wrong interpretation is text that is suddenly not ASCII at all. +/// +/// The cost is that a payload written in a non-Latin script falls back to raw hex rather than +/// being decoded. That is the conservative direction: raw hex is visibly unreadable, whereas +/// mojibake looks like data and misleads. Windows writes these traces in English. +fn readable_ratio(text: &str) -> f32 { + let total = text.chars().count(); + if total == 0 { + return 0.0; + } + let readable = text + .chars() + .filter(|c| matches!(c, ' '..='~' | '\n' | '\r' | '\t')) + .count(); + readable as f32 / total as f32 +} + +fn decode_utf16le(bytes: &[u8]) -> Option { + if !bytes.len().is_multiple_of(2) { + return None; + } + let units: Vec = bytes + .chunks_exact(2) + .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) + .collect(); + String::from_utf16(&units).ok() +} + +/// Text is accepted only when this much of it is readable. +/// +/// Set high because the alternative to accepting is showing the raw hex, which is at least +/// obviously unreadable. Mojibake looks like data and misleads. +const MINIMUM_READABLE_RATIO: f32 = 0.9; + +/// Decodes an `EventPayload` hex string into readable text. +/// +/// Returns `None` when the input is not hexadecimal, or when neither interpretation produces +/// convincingly readable text. The caller should then show the payload unchanged. +pub fn decode_event_payload(hex: &str) -> Option { + let bytes = hex_to_bytes(hex.trim())?; + + let mut best: Option<(String, f32, PayloadEncoding)> = None; + + if let Some(text) = decode_utf16le(&bytes) { + let trimmed = text.trim_end_matches('\0').to_string(); + let ratio = readable_ratio(&trimmed); + best = Some((trimmed, ratio, PayloadEncoding::Utf16Le)); + } + + if let Ok(text) = String::from_utf8(bytes) { + let trimmed = text.trim_end_matches('\0').to_string(); + let ratio = readable_ratio(&trimmed); + // Strictly greater, so ties go to UTF-16: Windows writes these payloads that way far more + // often, and a short ASCII payload decodes plausibly under both readings. + let better = match &best { + Some((_, best_ratio, _)) => ratio > *best_ratio, + None => true, + }; + if better { + best = Some((trimmed, ratio, PayloadEncoding::Utf8)); + } + } + + let (text, ratio, encoding) = best?; + if text.is_empty() || ratio < MINIMUM_READABLE_RATIO { + return None; + } + + Some(DecodedPayload { text, encoding }) +} + +/// The element name providers use for a hex-encoded message body. +const PAYLOAD_ELEMENT: &str = "EventPayload"; + +/// Finds and decodes the `EventPayload` element anywhere in a parsed event. +/// +/// The element's position varies: some providers put it under `UserData`, others under +/// `EventData`, and the wrapper element carries the provider's own name. Searching the whole tree +/// avoids hard-coding a path that would silently match nothing for half of them. +/// +/// The first decodable payload wins. Events carrying more than one are not something Windows +/// emits, and picking the first is at least deterministic. +pub fn decode_payload_in(root: &EventNode) -> Option { + if root.name == PAYLOAD_ELEMENT { + if let Some(decoded) = root.text.as_deref().and_then(decode_event_payload) { + return Some(decoded); + } + } + root.children.iter().find_map(decode_payload_in) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn utf16_hex(text: &str) -> String { + text.encode_utf16() + .flat_map(|unit| unit.to_le_bytes()) + .map(|byte| format!("{byte:02X}")) + .collect() + } + + fn utf8_hex(text: &str) -> String { + text.bytes().map(|byte| format!("{byte:02X}")).collect() + } + + #[test] + fn decodes_a_utf16le_payload() { + let decoded = + decode_event_payload(&utf16_hex("Agent * START * Finding updates")).expect("decodes"); + assert_eq!(decoded.text, "Agent * START * Finding updates"); + assert_eq!(decoded.encoding, PayloadEncoding::Utf16Le); + } + + #[test] + fn decodes_a_single_byte_payload() { + // Long enough that the UTF-16 reading is clearly worse, which is what decides it. + let text = "Downloading update 8f21b4c2 from the delivery service"; + let decoded = decode_event_payload(&utf8_hex(text)).expect("decodes"); + assert_eq!(decoded.text, text); + assert_eq!(decoded.encoding, PayloadEncoding::Utf8); + } + + #[test] + fn tolerates_whitespace_between_digits() { + assert_eq!( + decode_event_payload("48 00 69 00").expect("decodes").text, + "Hi" + ); + } + + #[test] + fn accepts_lowercase_hex() { + assert_eq!( + decode_event_payload(&utf16_hex("Hi").to_lowercase()) + .expect("decodes") + .text, + "Hi" + ); + } + + #[test] + fn strips_a_trailing_null_terminator() { + assert_eq!( + decode_event_payload(&utf16_hex("Done\0")) + .expect("decodes") + .text, + "Done" + ); + } + + #[test] + fn preserves_embedded_newlines() { + assert_eq!( + decode_event_payload(&utf16_hex("line one\r\nline two")) + .expect("decodes") + .text, + "line one\r\nline two" + ); + } + + #[test] + fn refuses_an_odd_number_of_digits() { + // Decoding half of it would present truncated evidence as complete. + assert!(decode_event_payload("48656C6C6").is_none()); + } + + #[test] + fn refuses_input_that_is_not_hexadecimal() { + assert!(decode_event_payload("not hex at all").is_none()); + assert!(decode_event_payload("ZZ00").is_none()); + assert!(decode_event_payload("").is_none()); + } + + #[test] + fn refuses_binary_that_is_not_text() { + // Both readings "succeed" here and neither is text: as UTF-8 these are control bytes, and + // as UTF-16 they become Latin Extended letters that look like language but are not. Showing + // either as a message would be a confident wrong answer; raw hex is at least honest. + let binary: String = (0u8..64).map(|byte| format!("{byte:02X}")).collect(); + assert!(decode_event_payload(&binary).is_none()); + } + + #[test] + fn refuses_a_payload_that_decodes_to_nothing() { + assert!(decode_event_payload("0000").is_none()); + } + + #[test] + fn a_short_ascii_payload_is_read_as_utf16_when_both_are_plausible() { + let decoded = decode_event_payload(&utf16_hex("OK")).expect("decodes"); + assert_eq!(decoded.text, "OK"); + assert_eq!(decoded.encoding, PayloadEncoding::Utf16Le); + } + + #[test] + fn finds_a_payload_nested_under_a_provider_wrapper() { + // The wrapper element carries the provider's own name, so the path cannot be hard-coded. + let root = EventNode::new("Event").with_child( + EventNode::new("UserData") + .with_child(EventNode::new("WindowsUpdateClient").with_child( + EventNode::new("EventPayload").with_text(utf16_hex("Agent ready")), + )), + ); + assert_eq!(decode_payload_in(&root).expect("found").text, "Agent ready"); + } + + #[test] + fn an_event_without_a_payload_yields_nothing() { + let root = EventNode::new("Event").with_child( + EventNode::new("EventData") + .with_child(EventNode::new("Data").with_text(utf16_hex("not a payload"))), + ); + assert!(decode_payload_in(&root).is_none()); + } + + #[test] + fn an_undecodable_payload_does_not_hide_a_later_decodable_one() { + // Refusing the first must not end the search, or one malformed element would suppress the + // readable message that follows it. + let root = EventNode::new("Event") + .with_child(EventNode::new("EventPayload").with_text("ZZZZ")) + .with_child(EventNode::new("EventPayload").with_text(utf16_hex("readable"))); + assert_eq!(decode_payload_in(&root).expect("found").text, "readable"); + } + + #[test] + fn round_trips_a_realistic_windows_update_line() { + let line = "2026-08-09 12:00:00.1234567 1234 5678 Agent Update service is being installed"; + assert_eq!( + decode_event_payload(&utf16_hex(line)) + .expect("decodes") + .text, + line + ); + } +} diff --git a/crates/cmtraceopen-parser/src/lib.rs b/crates/cmtraceopen-parser/src/lib.rs index 15702390a..439f09f75 100644 --- a/crates/cmtraceopen-parser/src/lib.rs +++ b/crates/cmtraceopen-parser/src/lib.rs @@ -11,6 +11,7 @@ pub mod collector; pub mod dsregcmd; pub mod error_db; pub mod esp; +pub mod event_payload; pub mod event_query; pub mod eventmap; pub mod intune; diff --git a/src-tauri/src/event_log/event_node.rs b/src-tauri/src/event_log/event_node.rs index f671bc18e..42abeaffd 100644 --- a/src-tauri/src/event_log/event_node.rs +++ b/src-tauri/src/event_log/event_node.rs @@ -137,6 +137,12 @@ fn local_name(raw: &[u8]) -> String { /// on every event, so they are extracted unconditionally. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct SystemFields { + pub provider: Option, + pub event_id: Option, + pub level: Option, + pub channel: Option, + pub computer: Option, + pub time_created: Option, pub task: Option, pub opcode: Option, pub process_id: Option, @@ -177,6 +183,14 @@ pub fn extract_system_fields(root: &EventNode) -> SystemFields { }; SystemFields { + provider: attribute_of("Provider", "Name"), + // Classic providers write `1000`. The id is the + // element text in both shapes; the qualifier is separate and not part of the id. + event_id: text_of("EventID").and_then(|value| value.parse().ok()), + level: text_of("Level").and_then(|value| value.parse().ok()), + channel: text_of("Channel").map(str::to_string), + computer: text_of("Computer").map(str::to_string), + time_created: attribute_of("TimeCreated", "SystemTime"), task: text_of("Task").and_then(|value| value.parse().ok()), opcode: text_of("Opcode").and_then(|value| value.parse().ok()), process_id: attribute_of("Execution", "ProcessID").and_then(|v| v.parse().ok()), @@ -328,7 +342,13 @@ mod tests { // Claiming task 0 when the provider wrote no Task element would be inventing evidence. let xml = "1"; let fields = extract_system_fields(&parse_event_xml(xml).expect("parses")); - assert_eq!(fields, SystemFields::default()); + assert_eq!( + fields, + SystemFields { + event_id: Some(1), + ..SystemFields::default() + } + ); } #[test] diff --git a/src-tauri/src/event_log/live.rs b/src-tauri/src/event_log/live.rs index c6c09ff91..11513bbf5 100644 --- a/src-tauri/src/event_log/live.rs +++ b/src-tauri/src/event_log/live.rs @@ -419,17 +419,34 @@ fn parse_xml_to_record( let timestamp = extract_xml_attr(xml, "TimeCreated", "SystemTime").unwrap_or_default(); let timestamp_epoch = parse_timestamp_to_epoch_ms(×tamp); - let event_data = extract_xml_event_data(xml); + let mut event_data = extract_xml_event_data(xml); - // Use rendered message if available, otherwise build summary from EventData. - // Sanitize to strip control characters that would show as unexpected glyphs. + // Parsed once and used for the System block, the decoded payload, and any registered map, so + // none of them costs an extra parse. + let parsed = super::event_node::parse_event_xml(xml).ok(); + + // Trace-backed channels carry their message as a hex blob rather than as EventData, so without + // this the row reads as a wall of hex digits. Surfaced as a field of its own because the raw + // hex never appears in EventData. + let payload = parsed + .as_ref() + .and_then(cmtraceopen_parser::event_payload::decode_payload_in) + .map(|decoded| sanitize_control_chars(&decoded.text)); + if let Some(text) = &payload { + event_data.push(EvtxField { + name: "EventPayload".to_string(), + value: text.clone(), + }); + } + + // The provider's own rendered message wins when there is one. A decoded payload comes next, + // used whole rather than through the summary, which would truncate the only text the event has + // at 80 characters. Sanitized to strip control characters that render as unexpected glyphs. let message = rendered_message .map(sanitize_control_chars) + .or(payload) .unwrap_or_else(|| build_event_data_summary(&event_data)); - // Parsed once and used for both the System block and any registered map, so mapping costs no - // extra parse. - let parsed = super::event_node::parse_event_xml(xml).ok(); let system = parsed .as_ref() .map(super::event_node::extract_system_fields) @@ -549,18 +566,6 @@ fn build_event_data_summary(fields: &[EvtxField]) -> String { .join("; ") } -/// Parse an ISO 8601 timestamp to epoch milliseconds. -fn parse_timestamp_to_epoch_ms(timestamp: &str) -> i64 { - chrono::DateTime::parse_from_rfc3339(timestamp) - .or_else(|_| { - // Windows timestamps may omit timezone, assume UTC - chrono::NaiveDateTime::parse_from_str(timestamp, "%Y-%m-%dT%H:%M:%S%.f") - .map(|naive| naive.and_utc().fixed_offset()) - }) - .map(|dt| dt.timestamp_millis()) - .unwrap_or(0) -} - // ── Error helpers ─────────────────────────────────────────────────────────── #[cfg(target_os = "windows")] diff --git a/src-tauri/src/event_log/mod.rs b/src-tauri/src/event_log/mod.rs index f7063a0aa..609b7bbd6 100644 --- a/src-tauri/src/event_log/mod.rs +++ b/src-tauri/src/event_log/mod.rs @@ -24,9 +24,48 @@ pub(crate) fn sanitize_control_chars(s: &str) -> String { .to_string() } +/// Parse an event timestamp to epoch milliseconds. +/// +/// Shared by the live and file paths so that a record sorts to the same place regardless of how it +/// was opened. Windows usually writes a full RFC 3339 stamp, but some providers omit the zone, in +/// which case it is read as UTC: the alternative is dropping the event to the epoch, which would +/// silently reorder the timeline rather than being off by a zone offset. +pub(crate) fn parse_timestamp_to_epoch_ms(timestamp: &str) -> i64 { + chrono::DateTime::parse_from_rfc3339(timestamp) + .or_else(|_| { + chrono::NaiveDateTime::parse_from_str(timestamp, "%Y-%m-%dT%H:%M:%S%.f") + .map(|naive| naive.and_utc().fixed_offset()) + }) + .map(|dt| dt.timestamp_millis()) + .unwrap_or(0) +} + #[cfg(test)] mod tests { - use super::sanitize_control_chars; + use super::{parse_timestamp_to_epoch_ms, sanitize_control_chars}; + + #[test] + fn reads_a_full_rfc3339_stamp() { + assert_eq!( + parse_timestamp_to_epoch_ms("2026-08-09T12:00:00.000Z"), + 1_786_276_800_000 + ); + } + + #[test] + fn a_stamp_without_a_zone_is_read_as_utc_rather_than_dropped() { + // Dropping it to zero would sort the event to 1970 and silently reorder the timeline. + assert_eq!( + parse_timestamp_to_epoch_ms("2026-08-09T12:00:00.000"), + parse_timestamp_to_epoch_ms("2026-08-09T12:00:00.000Z") + ); + } + + #[test] + fn an_unreadable_stamp_yields_zero() { + assert_eq!(parse_timestamp_to_epoch_ms("not a time"), 0); + assert_eq!(parse_timestamp_to_epoch_ms(""), 0); + } #[test] fn strips_trailing_carriage_return() { diff --git a/src-tauri/src/event_log/parser.rs b/src-tauri/src/event_log/parser.rs index 874fac7cb..60c4060c6 100644 --- a/src-tauri/src/event_log/parser.rs +++ b/src-tauri/src/event_log/parser.rs @@ -1,12 +1,11 @@ use std::path::Path; use evtx::EvtxParser; -use serde_json::Value; use super::models::{ ChannelSourceType, EvtxChannelInfo, EvtxField, EvtxLevel, EvtxParseResult, EvtxRecord, }; -use super::sanitize_control_chars; +use super::{parse_timestamp_to_epoch_ms, sanitize_control_chars}; /// Maximum entries to parse from a single .evtx file to prevent memory issues. const MAX_ENTRIES_PER_FILE: usize = 100_000; @@ -100,7 +99,10 @@ fn parse_single_file(path: &Path) -> Result<(Vec, u32), String> { let mut records = Vec::new(); let mut parse_errors = 0u32; - for record_result in parser.records_json_value() { + // XML rather than JSON. The JSON projection cannot be re-parsed into an event tree, which is + // what the map engine, the System block, and the XML export all consume; reading XML here is + // what makes those work on an opened file at all. + for record_result in parser.records() { if records.len() >= MAX_ENTRIES_PER_FILE { log::warn!( "event=evtx_entry_cap_reached file=\"{}\" cap={}", @@ -123,56 +125,56 @@ fn parse_single_file(path: &Path) -> Result<(Vec, u32), String> { } }; - let json = &record.data; - let system = &json["Event"]["System"]; - let event_data_val = &json["Event"]["EventData"]; - - let provider = system["Provider"]["#attributes"]["Name"] - .as_str() - .unwrap_or("Unknown") - .to_string(); - - let channel = system["Channel"].as_str().unwrap_or("Unknown").to_string(); - - let event_id = extract_event_id(system); - - let level = system["Level"].as_u64().unwrap_or(0) as u8; - let evtx_level = EvtxLevel::from_level_value(level); - - let computer = system["Computer"].as_str().unwrap_or("Unknown").to_string(); - - let timestamp_str = system["TimeCreated"]["#attributes"]["SystemTime"] - .as_str() - .unwrap_or("") - .to_string(); + let raw_xml = record.data; + let event_record_id = record.event_record_id; - let timestamp_epoch = chrono::DateTime::parse_from_rfc3339(×tamp_str) - .map(|dt| dt.timestamp_millis()) - .unwrap_or(0); + // Parsed once and used for identity, the System block, the decoded payload, and any + // registered map, so none of them costs an extra parse. A record whose XML will not parse + // is counted as an error rather than pushed with every field defaulted, which would show a + // row claiming provider "Unknown" at the epoch. + let parsed = match super::event_node::parse_event_xml(&raw_xml) { + Ok(root) => root, + Err(error) => { + log::warn!( + "event=evtx_record_unparsable file=\"{}\" error=\"{}\"", + path.display(), + error + ); + parse_errors += 1; + continue; + } + }; - let event_record_id = record.event_record_id; + let system = super::event_node::extract_system_fields(&parsed); + let provider = system.provider.clone().unwrap_or_else(|| "Unknown".into()); + let channel = system.channel.clone().unwrap_or_else(|| "Unknown".into()); + let event_id = system.event_id.unwrap_or(0); + let evtx_level = EvtxLevel::from_level_value(system.level.unwrap_or(0)); + let computer = system.computer.clone().unwrap_or_else(|| "Unknown".into()); + let timestamp_str = system.time_created.clone().unwrap_or_default(); + let timestamp_epoch = parse_timestamp_to_epoch_ms(×tamp_str); + + let mut event_data = extract_event_data(&parsed); + + // Same treatment as the live path: a trace-backed event carries its message as hex, and + // without decoding it the row is a wall of digits. + let payload = cmtraceopen_parser::event_payload::decode_payload_in(&parsed) + .map(|decoded| sanitize_control_chars(&decoded.text)); + if let Some(text) = &payload { + event_data.push(EvtxField { + name: "EventPayload".to_string(), + value: text.clone(), + }); + } - let event_data = extract_event_data(event_data_val); // A provider database, when one is loaded, turns raw field values into the sentence the // provider intended. Without it the file path can only summarise EventData, which is what // every other cross-platform reader shows and why they are hard to read. let message = describe_event(&provider, event_id, &event_data) + .or(payload) .unwrap_or_else(|| build_message(&event_data)); - // Build raw XML placeholder from JSON (actual XML not available via json_value API) - let raw_xml = serde_json::to_string_pretty(json).unwrap_or_default(); - - // Parsed once from the raw XML so both the live and file paths read the System block the - // same way, and so any registered map is applied without a second parse. - let parsed = super::event_node::parse_event_xml(&raw_xml).ok(); - let system = parsed - .as_ref() - .map(super::event_node::extract_system_fields) - .unwrap_or_default(); - let mapped = parsed - .as_ref() - .map(|root| super::maps::apply_global(&channel, &provider, event_id, root)) - .unwrap_or_default(); + let mapped = super::maps::apply_global(&channel, &provider, event_id, &parsed); records.push(EvtxRecord { id: 0, // Will be reassigned after sorting event_record_id, @@ -200,42 +202,50 @@ fn parse_single_file(path: &Path) -> Result<(Vec, u32), String> { Ok((records, parse_errors)) } -/// Extract EventID which can appear as `{"#text": N}` or just `N`. -fn extract_event_id(system: &Value) -> u32 { - if let Some(id) = system["EventID"].as_u64() { - return id as u32; - } - if let Some(id) = system["EventID"]["#text"].as_u64() { - return id as u32; - } - if let Some(s) = system["EventID"]["#text"].as_str() { - return s.parse().unwrap_or(0); - } - 0 -} - -/// Extract EventData fields as key-value pairs. +/// Extract event fields as name-value pairs. +/// +/// Both `EventData` and `UserData` are read. Manifest providers use the former and classic or +/// trace-backed providers the latter, and skipping `UserData` would leave those events with no +/// fields at all. /// -/// All values are sanitized to strip control characters (e.g. `\r`, `\0`) that -/// would render as unexpected glyphs in the UI. -fn extract_event_data(event_data: &Value) -> Vec { +/// A `Data` element with no `Name` attribute is numbered by its position, matching how the event +/// message template refers to it. Values are sanitized to strip control characters that would +/// render as unexpected glyphs. +fn extract_event_data(root: &cmtraceopen_parser::eventmap::EventNode) -> Vec { let mut fields = Vec::new(); + let mut unnamed = 0usize; - if let Some(obj) = event_data.as_object() { - for (key, value) in obj { - if key == "#attributes" { - continue; - } - let val_str = match value { - Value::String(s) => sanitize_control_chars(s), - Value::Null => continue, - other => sanitize_control_chars(&other.to_string()), - }; - if !val_str.is_empty() { - fields.push(EvtxField { - name: key.clone(), - value: val_str, - }); + let containers = root + .children + .iter() + .filter(|child| child.name == "EventData" || child.name == "UserData"); + + for container in containers { + // UserData wraps its fields in a provider-named element, so descend through a single + // wrapper when the container holds no Data of its own. + let holders: Vec<_> = if container.children.iter().any(|c| c.name == "Data") { + vec![container] + } else { + container.children.iter().collect() + }; + + for holder in holders { + for child in &holder.children { + let value = sanitize_control_chars(child.text.as_deref().unwrap_or_default()); + if value.is_empty() { + continue; + } + let name = match child.attribute("Name") { + Some(name) => name.to_string(), + // A positional field. Named from one so it lines up with the `%1` style + // insertion numbering that message templates use. + None if child.name == "Data" => { + unnamed += 1; + format!("Data{unnamed}") + } + None => child.name.clone(), + }; + fields.push(EvtxField { name, value }); } } } @@ -291,36 +301,109 @@ mod tests { assert_eq!(EvtxLevel::from_level_value(255), EvtxLevel::Information); } + fn parse(xml: &str) -> cmtraceopen_parser::eventmap::EventNode { + super::super::event_node::parse_event_xml(xml).expect("well formed") + } + + fn fields_of(xml: &str) -> Vec { + extract_event_data(&parse(xml)) + } + #[test] - fn test_extract_event_id_numeric() { - let json: Value = serde_json::json!({"EventID": 4624}); - assert_eq!(extract_event_id(&json), 4624); + fn named_event_data_becomes_named_fields() { + let fields = fields_of( + r#" + SYSTEM + 0x3e7 + "#, + ); + assert_eq!(fields.len(), 2); + assert_eq!(fields[0].name, "SubjectUserName"); + assert_eq!(fields[0].value, "SYSTEM"); + assert_eq!(fields[1].name, "TargetLogonId"); } #[test] - fn test_extract_event_id_text_object() { - let json: Value = serde_json::json!({"EventID": {"#text": 1001}}); - assert_eq!(extract_event_id(&json), 1001); + fn an_empty_data_element_is_dropped_rather_than_shown_blank() { + let fields = fields_of( + r#" + yes + + "#, + ); + assert_eq!(fields.len(), 1); + assert_eq!(fields[0].name, "Present"); } #[test] - fn test_extract_event_id_text_string() { - let json: Value = serde_json::json!({"EventID": {"#text": "999"}}); - assert_eq!(extract_event_id(&json), 999); + fn unnamed_data_is_numbered_from_one_to_match_insertion_order() { + // Classic providers emit positional Data. Numbering from one lines the fields up with the + // %1 style references in the provider's message template. + let fields = fields_of( + r#" + first + second + "#, + ); + assert_eq!(fields[0].name, "Data1"); + assert_eq!(fields[0].value, "first"); + assert_eq!(fields[1].name, "Data2"); + assert_eq!(fields[1].value, "second"); } #[test] - fn test_extract_event_data() { - let json: Value = serde_json::json!({ - "#attributes": {"Name": "test"}, - "SubjectUserName": "SYSTEM", - "TargetLogonId": "0x3e7" - }); - let fields = extract_event_data(&json); + fn user_data_fields_are_read_through_the_provider_wrapper() { + // Skipping UserData would leave every classic and trace-backed event with no fields at all. + let fields = fields_of( + r#" + + Enforce + C:\app.exe + + "#, + ); assert_eq!(fields.len(), 2); - assert!(fields - .iter() - .any(|f| f.name == "SubjectUserName" && f.value == "SYSTEM")); + assert_eq!(fields[0].name, "PolicyName"); + assert_eq!(fields[1].value, "C:\\app.exe"); + } + + #[test] + fn control_characters_in_a_value_are_stripped() { + let fields = fields_of( + "C:\\app.exe\r", + ); + assert_eq!(fields[0].value, "C:\\app.exe"); + } + + #[test] + fn system_identity_comes_off_the_parsed_tree() { + // The file path used to re-parse a JSON projection as XML, which always failed, leaving + // every System-derived column empty on an opened file. + let system = super::super::event_node::extract_system_fields(&parse( + r#" + + 12 + 2 + + System + RING0IVY24-01 + + "#, + )); + assert_eq!( + system.provider.as_deref(), + Some("Microsoft-Windows-Kernel-General") + ); + // The qualifier is a separate value; the id is still the element text. + assert_eq!(system.event_id, Some(12)); + assert_eq!(system.level, Some(2)); + assert_eq!(system.channel.as_deref(), Some("System")); + assert_eq!(system.computer.as_deref(), Some("RING0IVY24-01")); + assert_eq!(system.process_id, Some(4)); + assert_eq!( + parse_timestamp_to_epoch_ms(system.time_created.as_deref().unwrap_or_default()), + 1_786_276_800_000 + ); } #[test] From 70db7bdbe510e03c2ef3c811975969536d173de5 Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 9 Aug 2026 18:14:41 -0400 Subject: [PATCH 34/85] test(event-log): cover the .evtx file path against a real capture The unit tests exercise field extraction against hand-written XML. They could not have caught what actually broke: the path fed a JSON projection to an XML parser, so every record failed to parse and the System block, every map, and the XML export came back empty on an opened file. Only running a real file through the whole pipeline sees that. Captures carry real hostnames and query traffic, so the fixture stays outside the repo behind CMTRACE_EVTX_FIXTURE. Unset, every test passes vacuously and CI is unaffected. Assertions are floors and invariants so a different capture does not break them. Verified against a 2,121-event DNSServer/Audit log: zero parse errors, and PID, TID, task and keywords populated where they were previously all empty. Co-Authored-By: Claude Opus 5 --- src-tauri/tests/event_log_real_evtx.rs | 133 +++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 src-tauri/tests/event_log_real_evtx.rs diff --git a/src-tauri/tests/event_log_real_evtx.rs b/src-tauri/tests/event_log_real_evtx.rs new file mode 100644 index 000000000..6ace76eea --- /dev/null +++ b/src-tauri/tests/event_log_real_evtx.rs @@ -0,0 +1,133 @@ +//! Runs the .evtx file path against a real Windows event log. +//! +//! The unit tests in `event_log::parser` cover field extraction against hand-written XML. They +//! cannot cover the part that actually broke: what the `evtx` crate hands back per record, and +//! whether that survives the whole pipeline. The file path once fed a JSON projection to an XML +//! parser, which failed silently for every record and left the System block, every map, and the +//! XML export empty on an opened file. Nothing in the unit tests could see that. +//! +//! Captured logs carry real hostnames, account names and query traffic, so they live outside the +//! repo. Point `CMTRACE_EVTX_FIXTURE` at an .evtx file to run these; without it every test here +//! passes vacuously, so CI is unaffected. +//! +//! CMTRACE_EVTX_FIXTURE=~/logs/dns-audit.evtx cargo test --features event-log --test event_log_real_evtx +//! +//! Assertions are floors and invariants rather than exact counts, so a different capture does not +//! break them. + +#![cfg(feature = "event-log")] + +use std::path::PathBuf; + +use app_lib::event_log::export::{export_records, ExportFormat}; +use app_lib::event_log::models::EvtxParseResult; +use app_lib::event_log::parser::parse_evtx_files; + +fn fixture() -> Option { + let path = PathBuf::from(std::env::var_os("CMTRACE_EVTX_FIXTURE")?); + assert!( + path.is_file(), + "CMTRACE_EVTX_FIXTURE is set but {} is not a file", + path.display() + ); + Some(path) +} + +fn parsed() -> Option { + let path = fixture()?; + let result = parse_evtx_files(&[path.to_string_lossy().into_owned()]).expect("the file parses"); + assert!( + !result.records.is_empty(), + "fixture produced no records at all" + ); + Some(result) +} + +#[test] +fn every_record_in_the_file_parses() { + let Some(result) = parsed() else { return }; + // A record that cannot be read is counted, not dropped in silence. A real log from a healthy + // machine should have none, so any error here means the reader is wrong rather than the file. + assert_eq!( + result.parse_errors, + 0, + "{} of {} records failed to parse", + result.parse_errors, + result.records.len() + ); +} + +#[test] +fn identity_is_populated_rather_than_defaulted() { + let Some(result) = parsed() else { return }; + for record in &result.records { + assert_ne!( + record.provider, "Unknown", + "record {} has no provider", + record.event_record_id + ); + assert_ne!( + record.channel, "Unknown", + "record {} has no channel", + record.event_record_id + ); + assert_ne!(record.event_id, 0, "record has no event id"); + assert_ne!( + record.timestamp_epoch, 0, + "record {} sorted to the epoch, so the timeline order is wrong", + record.event_record_id + ); + } +} + +#[test] +fn the_system_block_survives_the_file_path() { + let Some(result) = parsed() else { return }; + // These were empty on every opened file while the path re-parsed JSON as XML. Asserting that + // some record carries each one, rather than all of them, because providers legitimately omit + // any individual field. + let has = |f: fn(&app_lib::event_log::models::EvtxRecord) -> bool| result.records.iter().any(f); + assert!(has(|r| r.process_id.is_some()), "no record carries a PID"); + assert!(has(|r| r.thread_id.is_some()), "no record carries a TID"); + assert!(has(|r| r.keywords.is_some()), "no record carries keywords"); +} + +#[test] +fn records_carry_event_fields() { + let Some(result) = parsed() else { return }; + assert!( + result.records.iter().any(|r| !r.event_data.is_empty()), + "no record has any fields, so EventData and UserData were both missed" + ); + for record in &result.records { + for field in &record.event_data { + assert!(!field.name.is_empty(), "a field was extracted with no name"); + } + } +} + +#[test] +fn the_xml_export_emits_the_provider_representation() { + let Some(result) = parsed() else { return }; + // The export used to contain pretty-printed JSON under an root, which no XML consumer + // could read. + let exported = export_records(&result.records[..1], ExportFormat::Xml).expect("exports"); + assert!( + exported.contains(" Date: Sun, 9 Aug 2026 18:30:35 -0400 Subject: [PATCH 35/85] fix(parser): give every split query node an Id and pin the structured form The bare XPath forms were pinned against the live service from the start. The structured form never was, and it is the one that only appears once a filter outgrows the expression budget, so it would have reached a user unverified. CodeRabbit flagged the missing Id against the published schema, which calls it required once a list holds more than one Query. Measured rather than assumed, on Windows 11 build 26200 and deliberately without EvtQueryTolerateQueryErrors, since that flag makes the service accept a query whose nodes it could not evaluate and quietly return the rest: - a two-node list with no Id returned 5240 events, exactly matching the single expression covering the same IDs (5180 + 60), so the service does not in fact enforce the schema here - ten nodes behaved the same, as did duplicate Ids - all four builder-emitted shapes were accepted and returned nonzero counts, which rules out a query that silently matches nothing So this is not the defect it was reported as. The Id is written anyway because it costs nothing and a saved custom view is validated against the same schema. No node names a Path: EvtQuery supplies the channel from its own argument, and the schema requires that if any node names a path they all do. Five tests pin the validated shapes, including that every node repeats the other predicates, since the service unions the nodes rather than intersecting them. Co-Authored-By: Claude Opus 5 --- .../cmtraceopen-parser/src/event_query/mod.rs | 132 +++++++++++++++++- 1 file changed, 129 insertions(+), 3 deletions(-) diff --git a/crates/cmtraceopen-parser/src/event_query/mod.rs b/crates/cmtraceopen-parser/src/event_query/mod.rs index edadf01a9..41c452e86 100644 --- a/crates/cmtraceopen-parser/src/event_query/mod.rs +++ b/crates/cmtraceopen-parser/src/event_query/mod.rs @@ -354,12 +354,24 @@ pub fn build_query(filter: &EventQueryFilter) -> Result } let mut query = String::from(""); - for chunk in chunk_by_expression_budget(&filter.event_ids, fixed_cost) { + for (id, chunk) in chunk_by_expression_budget(&filter.event_ids, fixed_cost) + .iter() + .enumerate() + { + // The schema documents Id as required once the list holds more than one Query. The service + // does not enforce it: a two-node list without Id was measured returning exactly the same + // events as the equivalent single expression. It is written anyway because it costs + // nothing and the same document shape is what a saved custom view is validated against. + // + // No Path is written. EvtQuery supplies the channel from its own argument when the + // document omits it, and the schema requires that if any node names a path they all do, so + // omitting it everywhere is the consistent choice. + // // The expression becomes XML text here, so it is escaped at exactly this boundary. let _ = write!( query, - "", - escape_for_xml(&select_body(filter, &chunk)?) + "", + escape_for_xml(&select_body(filter, chunk)?) ); } query.push_str(""); @@ -984,3 +996,117 @@ mod expression_budget_tests { assert_eq!(query.matches("*[System[(EventID=1000 or EventID=1001 or EventID=1002 or \ + EventID=1003 or EventID=1004 or EventID=1005 or EventID=1006 or EventID=1007 or \ + EventID=1008 or EventID=1009 or EventID=1010 or EventID=1011 or EventID=1012 or \ + EventID=1013 or EventID=1014 or EventID=1015 or EventID=1016 or EventID=1017 or \ + EventID=1018 or EventID=1019)]]\ + \ + " + ); + } + + #[test] + fn every_node_carries_a_unique_id() { + let filter = EventQueryFilter { + event_ids: ids(100), + ..Default::default() + }; + let query = build_query(&filter).expect("builds"); + let node_count = query.matches("")).count(), + 1, + "id {id} is missing or repeated" + ); + } + } + + #[test] + fn no_node_names_a_channel_path() { + // EvtQuery supplies the channel from its own argument, and the schema requires that if any + // node names a path they all do. Omitting it everywhere is the consistent choice, and was + // accepted by the service. + let filter = EventQueryFilter { + event_ids: ids(31), + ..Default::default() + }; + assert!(!build_query(&filter).expect("builds").contains("Path=")); + } + + #[test] + fn operators_inside_a_structured_query_are_xml_escaped() { + // The inverse of the bare-XPath rule. Raw operators here would not be well-formed XML, and + // escaped operators in a bare XPath are rejected outright. + let filter = EventQueryFilter { + event_ids: ids(31), + levels: vec![1, 2, 3, 4], + time: Some(TimeWindow::Last { + milliseconds: 2_592_000_000, + }), + ..Default::default() + }; + let query = build_query(&filter).expect("builds"); + assert!(query.contains("timediff(@SystemTime) <= 2592000000")); + assert!(!query.contains("<= 2592000000")); + } + + #[test] + fn the_other_predicates_repeat_in_every_node() { + // The service unions the nodes rather than intersecting them, so a predicate that appears + // in only one node would widen the result set instead of narrowing it. + let filter = EventQueryFilter { + event_ids: ids(31), + keywords: Some(0x8020_0000_0000_0000), + ..Default::default() + }; + let query = build_query(&filter).expect("builds"); + assert_eq!( + query.matches("band").count(), + query.matches("` nodes inside one ``. //! FullEventLogView batches at ten per node; the same bound is used here. //! - **Values are attacker-influenced.** Provider names reach this from user input and from event -//! data, so an apostrophe is stripped. It delimits XPath string literals and has no XPath 1.0 -//! escape, so leaving it in would terminate the literal and let the rest be read as syntax. +//! data, and an apostrophe delimits XPath string literals with no escape available. Rather than +//! strip it, which would silently turn `Bob's Agent` into a filter that matches nothing, the +//! value is quoted with whichever delimiter it does not contain. A value holding both is refused +//! as `UnquotableValue`, because there is no correct way to express it. //! - **Escaping depends on context, and getting it backwards fails closed.** A bare XPath must use //! raw `<=` and `>=`; the same operators inside a `` document must be XML-escaped. //! Verified against the service on Windows 11: escaped operators in a bare XPath are rejected @@ -145,12 +147,29 @@ impl EventQueryFilter { /// Expression budget for one `").skip(1) { + let body = node.split("").next().unwrap_or_default(); + // Each comparison is one expression; they are joined by `and` or `or`. + let comparisons = body.matches("EventID").count() + + body.matches("Level=").count() + + body.matches("@SystemTime").count() + + body.matches("timediff").count() + + body.matches("band").count(); + assert!( + comparisons <= MAX_EXPRESSIONS_PER_SELECT, + "{comparisons} expressions in one node for {id_count} ids, {levels:?} levels" + ); + } + } + } + } +} diff --git a/crates/cmtraceopen-parser/src/unified_timeline/mod.rs b/crates/cmtraceopen-parser/src/unified_timeline/mod.rs index 0e7246cb2..70917170e 100644 --- a/crates/cmtraceopen-parser/src/unified_timeline/mod.rs +++ b/crates/cmtraceopen-parser/src/unified_timeline/mod.rs @@ -75,8 +75,11 @@ pub enum TimelineOrigin { }, /// A Windows event. Event { + /// Channel the event was read from, for example `Microsoft-Windows-DNSServer/Audit`. channel: String, + /// Publisher that raised it, as the event's own `System` block names it. provider: String, + /// Event ID, which identifies the event only in combination with the provider. event_id: u32, /// Record identifier within its channel. record_id: u64, @@ -89,8 +92,11 @@ pub enum TimelineOrigin { pub struct TimelineItem { /// Milliseconds since the Unix epoch. pub timestamp_ms: i64, + /// How serious the source said this was, normalized across event levels and log levels. pub severity: TimelineSeverity, + /// The rendered text, already whatever the source's own formatting produced. pub message: String, + /// Which file or channel this came from, so a row on a merged timeline stays attributable. pub origin: TimelineOrigin, } @@ -134,10 +140,17 @@ impl UnifiedTimeline { } /// Inclusive time span covered, or `None` when nothing was placed. + /// + /// The bounds are computed rather than read off the ends. `merge` sorts, so reading the ends + /// would be correct on that path, but `items` is a public field and a value built directly can + /// hold items in any order. Reporting a span narrower than the data would understate what a + /// timeline actually covers. pub fn span_ms(&self) -> Option<(i64, i64)> { - let first = self.items.first()?.timestamp_ms; - let last = self.items.last()?.timestamp_ms; - Some((first, last)) + let mut stamps = self.items.iter().map(|item| item.timestamp_ms); + let first = stamps.next()?; + Some(stamps.fold((first, first), |(low, high), stamp| { + (low.min(stamp), high.max(stamp)) + })) } } From 9ce3e9021f32ed2232a5da484bb5deced5557cee Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 9 Aug 2026 22:12:30 -0400 Subject: [PATCH 45/85] fix(event-log): stop misattributing values in rendered descriptions Seven review findings. The first is the one that mattered. A provider's description template addresses fields by position. Field extraction dropped values the provider left empty, and the same filtered list was handed to the renderer as the insertion list, so an empty field shifted every later reference: %3 resolved to what %4 said and the description stated it as fact. The display list and the insertion list are now built separately, because they genuinely differ. A blank field is noise in a column and load bearing in a template. Also fixed: - keyword_names returned names in the map's own key order, which is lexicographic over decimal strings: "1", "16", "2", "32", "4". The order matched neither the mask nor the provider's manifest while looking as though it meant something. Sorted by bit value. The old test could not catch it because it used only single-digit keys. - the provider database opened with SQLITE_OPEN_URI, so any path beginning "file:" was parsed as a URI and its parameters honoured, vfs= included. These paths come from scanning a directory the operator chose, so a file dropped there could pick how SQLite opened it. The flag is gone. - group keys embedded values unencoded, so a provider literally named "x/level=Error" collided with the Error subgroup of provider "x" and collapsing one collapsed the other. - the severity mapping test asserted only the timestamp inside its loop, which is independent of the level, so a wrong arm for Warning, Error or Information passed. Every arm is asserted now. - the time-filter service test queried the wide window first, leaving a gap in which a new event lands inside the narrow result and outside the wide one already collected. Narrow runs first, so a new event falls on the side the assertion tolerates. - the fixture-gated integration tests printed ok when the fixture was absent, which is the same "empty result reads as a verified one" failure the suite exists to catch. It now says it skipped. Co-Authored-By: Claude Opus 5 --- crates/cmtraceopen-parser/src/provider/mod.rs | 63 ++++++++-- src-tauri/src/event_log/live.rs | 17 ++- src-tauri/src/event_log/parser.rs | 110 ++++++++++++++---- src-tauri/src/event_log/provider_db.rs | 6 +- src-tauri/src/event_log/timeline.rs | 22 ++-- src-tauri/tests/event_log_real_evtx.rs | 12 +- src/workspaces/event-log/evtx-filter.ts | 4 +- .../event-log/evtx-grouping.test.ts | 46 ++++++++ 8 files changed, 230 insertions(+), 50 deletions(-) diff --git a/crates/cmtraceopen-parser/src/provider/mod.rs b/crates/cmtraceopen-parser/src/provider/mod.rs index 4d7c19070..5ce5e3a86 100644 --- a/crates/cmtraceopen-parser/src/provider/mod.rs +++ b/crates/cmtraceopen-parser/src/provider/mod.rs @@ -148,15 +148,20 @@ impl ProviderMetadata { /// Only bits the provider declares are named. Undeclared bits are ignored rather than reported /// as unknown keywords, because the reserved high bits are set by the system on most events. pub fn keyword_names(&self, mask: u64) -> Vec<&str> { - let mut names = Vec::new(); - for (raw_bit, name) in &self.keywords { - if let Ok(bit) = raw_bit.parse::() { - if bit != 0 && mask & bit == bit { - names.push(name.as_str()); - } - } - } - names + // Sorted by bit value, not by key. The map is keyed by the decimal bit as a string, so its + // own order is lexicographic: "1", "16", "2", "32", "4". Returning that would put the names + // in an order matching neither the mask nor the provider's manifest, which reads as + // meaningful when it is an artefact of string comparison. + let mut matched: Vec<(u64, &str)> = self + .keywords + .iter() + .filter_map(|(raw_bit, name)| { + let bit = raw_bit.parse::().ok()?; + (bit != 0 && mask & bit == bit).then_some((bit, name.as_str())) + }) + .collect(); + matched.sort_by_key(|(bit, _)| *bit); + matched.into_iter().map(|(_, name)| name).collect() } } @@ -449,4 +454,44 @@ mod tests { assert_eq!(meta.source_os_build, Some(26200)); assert_eq!(meta.task_name(1), Some("None")); } + + #[test] + fn keyword_names_come_back_in_bit_order_not_key_order() { + // The map is keyed by the decimal bit as a string, so its own order is "1", "16", "2", + // "32", "4". Returning that reads as meaningful when it is an artefact of string sorting. + let mut keywords = std::collections::BTreeMap::new(); + for (bit, name) in [ + ("1", "Startup"), + ("2", "Shutdown"), + ("4", "Network"), + ("16", "Disk"), + ("32", "Memory"), + ] { + keywords.insert(bit.to_string(), name.to_string()); + } + let metadata = ProviderMetadata { + keywords, + ..Default::default() + }; + + assert_eq!( + metadata.keyword_names(0b11_0111), + vec!["Startup", "Shutdown", "Network", "Disk", "Memory"] + ); + } + + #[test] + fn only_the_bits_present_in_the_mask_are_named() { + let mut keywords = std::collections::BTreeMap::new(); + keywords.insert("1".to_string(), "Startup".to_string()); + keywords.insert("16".to_string(), "Disk".to_string()); + keywords.insert("32".to_string(), "Memory".to_string()); + let metadata = ProviderMetadata { + keywords, + ..Default::default() + }; + + assert_eq!(metadata.keyword_names(0b10_0001), vec!["Startup", "Memory"]); + assert!(metadata.keyword_names(0).is_empty()); + } } diff --git a/src-tauri/src/event_log/live.rs b/src-tauri/src/event_log/live.rs index c1e453147..3953065ca 100644 --- a/src-tauri/src/event_log/live.rs +++ b/src-tauri/src/event_log/live.rs @@ -668,29 +668,34 @@ mod live_service_tests { #[test] #[ignore = "requires a live Windows Event Log service with events"] fn a_time_filter_is_applied_by_the_service_and_narrows_the_result() { - let wide = query_channel_filtered( + // The narrow window is queried first. Running the wide one first leaves a gap in which a + // newly written event lands inside the one-hour result and outside the thirty-day one + // already collected, failing the assertion for a reason that has nothing to do with + // filtering. Querying narrow first makes any new event land in the wide result, which is + // the direction the assertion tolerates. + let narrow = query_channel_filtered( CHANNEL, &EventQueryFilter { time: Some(TimeWindow::Last { - milliseconds: 30 * 24 * 60 * 60 * 1000, + milliseconds: 60 * 60 * 1000, }), ..Default::default() }, None, ) - .expect("30 day query succeeds"); + .expect("1 hour query succeeds"); - let narrow = query_channel_filtered( + let wide = query_channel_filtered( CHANNEL, &EventQueryFilter { time: Some(TimeWindow::Last { - milliseconds: 60 * 60 * 1000, + milliseconds: 30 * 24 * 60 * 60 * 1000, }), ..Default::default() }, None, ) - .expect("1 hour query succeeds"); + .expect("30 day query succeeds"); assert!( narrow.len() <= wide.len(), diff --git a/src-tauri/src/event_log/parser.rs b/src-tauri/src/event_log/parser.rs index 8caf28ce2..84d1d6176 100644 --- a/src-tauri/src/event_log/parser.rs +++ b/src-tauri/src/event_log/parser.rs @@ -178,14 +178,18 @@ fn parse_single_file(path: &Path) -> Result { let timestamp_str = system.time_created.clone().unwrap_or_default(); let timestamp_epoch = parse_timestamp_to_epoch_ms(×tamp_str); - let mut event_data = extract_event_data(&parsed); + let EventFields { + mut fields, + insertions, + } = extract_event_data(&parsed); // Same treatment as the live path: a trace-backed event carries its message as hex, and // without decoding it the row is a wall of digits. let payload = cmtraceopen_parser::event_payload::decode_payload_in(&parsed) .map(|decoded| sanitize_control_chars(&decoded.text)); if let Some(text) = &payload { - event_data.push(EvtxField { + // Appended after every real field, so it cannot disturb the positional insertions. + fields.push(EvtxField { name: "EventPayload".to_string(), value: text.clone(), }); @@ -194,9 +198,9 @@ fn parse_single_file(path: &Path) -> Result { // A provider database, when one is loaded, turns raw field values into the sentence the // provider intended. Without it the file path can only summarise EventData, which is what // every other cross-platform reader shows and why they are hard to read. - let message = describe_event(&provider, event_id, &event_data) + let message = describe_event(&provider, event_id, &insertions) .or(payload) - .unwrap_or_else(|| build_message(&event_data)); + .unwrap_or_else(|| build_message(&fields)); let mapped = super::maps::apply_global(&channel, &provider, event_id, &parsed); records.push(EvtxRecord { @@ -210,7 +214,7 @@ fn parse_single_file(path: &Path) -> Result { level: evtx_level, computer, message, - event_data, + event_data: fields, raw_xml, source_label: source_label.clone(), task: system.task, @@ -254,8 +258,9 @@ fn parse_single_file(path: &Path) -> Result { /// A `Data` element with no `Name` attribute is numbered by its position, matching how the event /// message template refers to it. Values are sanitized to strip control characters that would /// render as unexpected glyphs. -fn extract_event_data(root: &cmtraceopen_parser::eventmap::EventNode) -> Vec { +fn extract_event_data(root: &cmtraceopen_parser::eventmap::EventNode) -> EventFields { let mut fields = Vec::new(); + let mut insertions = Vec::new(); let mut unnamed = 0usize; let containers = root @@ -271,8 +276,13 @@ fn extract_event_data(root: &cmtraceopen_parser::eventmap::EventNode) -> Vec, + insertions: &mut Vec, unnamed: &mut usize| { let value = sanitize_control_chars(child.text.as_deref().unwrap_or_default()); + // Recorded even when empty. The provider's message template addresses fields by position, + // so skipping one here shifts every later %N and renders the description with the wrong + // values substituted into it, which reads as fact. + insertions.push(value.clone()); if value.is_empty() { return; } @@ -292,16 +302,26 @@ fn extract_event_data(root: &cmtraceopen_parser::eventmap::EventNode) -> Vec, + insertions: Vec, } /// Renders the provider's own description for this event, when metadata for it is loaded. @@ -313,13 +333,12 @@ fn extract_event_data(root: &cmtraceopen_parser::eventmap::EventNode) -> Vec Option { +fn describe_event(provider: &str, event_id: u32, insertions: &[String]) -> Option { let metadata = super::provider_db::provider(provider)?; let event = metadata.event(event_id, None)?; let template = event.description.as_deref()?; - let insertions: Vec = event_data.iter().map(|field| field.value.clone()).collect(); - let rendered = cmtraceopen_parser::provider::render_description(template, &insertions); + let rendered = cmtraceopen_parser::provider::render_description(template, insertions); if rendered.is_complete() { Some(super::sanitize_control_chars(&rendered.text)) } else { @@ -357,7 +376,11 @@ mod tests { } fn fields_of(xml: &str) -> Vec { - extract_event_data(&parse(xml)) + extract_event_data(&parse(xml)).fields + } + + fn insertions_of(xml: &str) -> Vec { + extract_event_data(&parse(xml)).insertions } #[test] @@ -442,6 +465,41 @@ mod tests { assert_eq!(fields[1].value, "C:\\app.exe"); } + #[test] + fn an_empty_field_still_holds_its_insertion_position() { + // The provider's template addresses fields by position. Dropping the empty one would make + // %3 resolve to what %4 said, and the rendered description would state it as fact. + let xml = r#" + alpha + + gamma + "#; + + assert_eq!(insertions_of(xml), vec!["alpha", "", "gamma"]); + // The display list still omits the blank, because a column of blanks is noise. + assert_eq!( + fields_of(xml) + .iter() + .map(|f| f.name.as_str()) + .collect::>(), + vec!["First", "Third"] + ); + } + + #[test] + fn a_leading_empty_field_does_not_shift_the_rest() { + let xml = "second"; + assert_eq!(insertions_of(xml), vec!["", "second"]); + } + + #[test] + fn insertions_cover_user_data_too() { + let xml = r#" + onethree + "#; + assert_eq!(insertions_of(xml), vec!["one", "", "three"]); + } + #[test] fn a_binary_only_event_keeps_its_value() { // Classic providers emit with no at all. Treating a container without @@ -554,26 +612,27 @@ mod tests { mod description_tests { use super::*; - fn fields(values: &[(&str, &str)]) -> Vec { + /// Positional insertions, which is what a description template consumes. + /// + /// Names are kept in the call sites for readability but are not what the template addresses; + /// it refers to fields by position, which is why the insertion list carries empties. + fn insertions(values: &[(&str, &str)]) -> Vec { values .iter() - .map(|(name, value)| EvtxField { - name: name.to_string(), - value: value.to_string(), - }) + .map(|(_name, value)| value.to_string()) .collect() } #[test] fn with_no_database_loaded_it_falls_back_to_the_field_summary() { // The common case until an operator loads metadata. Must not fail or blank the message. - let data = fields(&[("HRESULT", "0x80180005")]); + let data = insertions(&[("HRESULT", "0x80180005")]); assert!(describe_event("Nobody-Has-This-Provider", 1, &data).is_none()); } #[test] fn an_unknown_event_id_falls_back_rather_than_inventing_a_description() { - let data = fields(&[("X", "1")]); + let data = insertions(&[("X", "1")]); assert!(describe_event("Still-Not-Loaded", 999_999, &data).is_none()); } @@ -587,7 +646,7 @@ mod description_tests { .expect("database has a parent directory"); super::super::provider_db::load_directory(directory).expect("databases load"); - let data = fields(&[("HRESULT", "0x80180005")]); + let data = insertions(&[("HRESULT", "0x80180005")]); let described = describe_event( "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider", 2, @@ -612,8 +671,11 @@ mod description_tests { super::super::provider_db::load_directory(directory).expect("databases load"); // A provider that genuinely is not in a Windows capture. - assert!( - describe_event("Definitely-Not-A-Real-Provider", 1, &fields(&[("a", "b")])).is_none() - ); + assert!(describe_event( + "Definitely-Not-A-Real-Provider", + 1, + &insertions(&[("a", "b")]) + ) + .is_none()); } } diff --git a/src-tauri/src/event_log/provider_db.rs b/src-tauri/src/event_log/provider_db.rs index 154631a54..b14f8e5d1 100644 --- a/src-tauri/src/event_log/provider_db.rs +++ b/src-tauri/src/event_log/provider_db.rs @@ -86,7 +86,11 @@ impl ProviderDb { pub fn open(path: &Path) -> Result { let connection = Connection::open_with_flags( path, - OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_URI, + // Read-only, and deliberately without SQLITE_OPEN_URI. With that flag any path + // starting with "file:" is parsed as a URI and its parameters honoured, including + // vfs=. These paths come from scanning a directory the operator pointed at, so a + // file dropped there could otherwise choose how SQLite opens it. + OpenFlags::SQLITE_OPEN_READ_ONLY, ) .map_err(|error| format!("cannot open provider database {}: {error}", path.display()))?; diff --git a/src-tauri/src/event_log/timeline.rs b/src-tauri/src/event_log/timeline.rs index 203806a0e..a35450349 100644 --- a/src-tauri/src/event_log/timeline.rs +++ b/src-tauri/src/event_log/timeline.rs @@ -187,18 +187,24 @@ mod tests { #[test] fn every_level_maps_without_panicking() { - for level in [ - EvtxLevel::Critical, - EvtxLevel::Error, - EvtxLevel::Warning, - EvtxLevel::Information, - EvtxLevel::Verbose, + // Every arm asserted, not just the two ends. The loop previously only checked the + // timestamp, which is independent of the level, so a wrong arm for Warning, Error or + // Information passed. + for (level, expected) in [ + (EvtxLevel::Critical, TimelineSeverity::Critical), + (EvtxLevel::Error, TimelineSeverity::Error), + (EvtxLevel::Warning, TimelineSeverity::Warning), + (EvtxLevel::Information, TimelineSeverity::Info), + (EvtxLevel::Verbose, TimelineSeverity::Verbose), ] { let item = from_event(&record(1, "x", level)).expect("placed"); assert_eq!(item.timestamp_ms, 1); + assert_eq!( + item.severity, expected, + "{level:?} must map to {expected:?}" + ); + assert_eq!(severity_of(level), expected); } - assert_eq!(severity_of(EvtxLevel::Critical), TimelineSeverity::Critical); - assert_eq!(severity_of(EvtxLevel::Verbose), TimelineSeverity::Verbose); } #[test] diff --git a/src-tauri/tests/event_log_real_evtx.rs b/src-tauri/tests/event_log_real_evtx.rs index 0ceb72dc4..a41920730 100644 --- a/src-tauri/tests/event_log_real_evtx.rs +++ b/src-tauri/tests/event_log_real_evtx.rs @@ -24,7 +24,17 @@ use app_lib::event_log::models::EvtxParseResult; use app_lib::event_log::parser::parse_evtx_files; fn fixture() -> Option { - let path = PathBuf::from(std::env::var_os("CMTRACE_EVTX_FIXTURE")?); + let Some(raw) = std::env::var_os("CMTRACE_EVTX_FIXTURE") else { + // Said out loud rather than passing silently. Seven tests reporting ok with nothing run is + // the same failure this suite exists to catch: an empty result that reads as a verified + // one. Visible with `cargo test -- --nocapture`, and in CI logs. + eprintln!( + "SKIP: CMTRACE_EVTX_FIXTURE is not set, so nothing in this file actually ran. \ + Point it at an .evtx file to exercise the real parse path." + ); + return None; + }; + let path = PathBuf::from(raw); assert!( path.is_file(), "CMTRACE_EVTX_FIXTURE is set but {} is not a file", diff --git a/src/workspaces/event-log/evtx-filter.ts b/src/workspaces/event-log/evtx-filter.ts index ef577fc9a..5c71fd147 100644 --- a/src/workspaces/event-log/evtx-filter.ts +++ b/src/workspaces/event-log/evtx-filter.ts @@ -163,7 +163,9 @@ export function buildGroupedRows( } for (const [value, bucket] of buckets) { - const key = `${parentKey}/${field}=${value}`; + // Encoded, so a value containing the delimiters cannot forge another group's ancestry. Two + // distinct paths sharing a key would make collapsing one collapse the other. + const key = `${parentKey}/${field}=${encodeURIComponent(value)}`; const collapsed = collapsedKeys.has(key); rows.push({ kind: "group", diff --git a/src/workspaces/event-log/evtx-grouping.test.ts b/src/workspaces/event-log/evtx-grouping.test.ts index a9a01c62e..57476cc64 100644 --- a/src/workspaces/event-log/evtx-grouping.test.ts +++ b/src/workspaces/event-log/evtx-grouping.test.ts @@ -158,3 +158,49 @@ describe("allGroupKeys", () => { expect(allGroupKeys([record({})], []).size).toBe(0); }); }); + +describe("group key encoding", () => { + it("keeps distinct ancestries distinct when a value contains the delimiters", () => { + // Unencoded, a provider literally named "x/level=Error" produces the same key as the + // Error subgroup of provider "x", and collapsing one would collapse the other. + const rows = buildGroupedRows( + [ + record({ id: 1, provider: "x/level=Error", level: "Information" }), + record({ id: 2, provider: "x", level: "Error" }), + ], + ["provider", "level"], + new Set() + ); + + const keys = rows.flatMap((r) => (r.kind === "group" ? [r.key] : [])); + expect(new Set(keys).size).toBe(keys.length); + }); + + it("collapsing one group does not collapse an unrelated one", () => { + const all = buildGroupedRows( + [ + record({ id: 1, provider: "x/level=Error", level: "Information" }), + record({ id: 2, provider: "x", level: "Error" }), + ], + ["provider", "level"], + new Set() + ); + const target = all.find((r) => r.kind === "group" && r.label === "Error"); + expect(target?.kind).toBe("group"); + const targetKey = target && target.kind === "group" ? target.key : ""; + + const collapsed = buildGroupedRows( + [ + record({ id: 1, provider: "x/level=Error", level: "Information" }), + record({ id: 2, provider: "x", level: "Error" }), + ], + ["provider", "level"], + new Set([targetKey]) + ); + + // Exactly one record disappears; the other ancestry is untouched. + const visible = collapsed.filter((r) => r.kind === "record"); + expect(visible).toHaveLength(1); + expect(visible[0].record.id).toBe(1); + }); +}); From 55b0294e977ab1598fd4365e31868f98980674e1 Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 9 Aug 2026 22:30:29 -0400 Subject: [PATCH 46/85] refactor(event-log): own the map and provider registries in AppState Both were process-wide OnceLock globals. The repository rule is that shared state reaches commands through Tauri's managed AppState, and two concrete things followed from breaking it. Tests shared one set. load_directory and load_global each clear and replace, and cargo runs tests on parallel threads in one process, so a test registering databases could replace the set another test was mid-way through asserting on. Nothing serialized them; it had not bitten yet only because the tests that mutate are mostly ignored by default. Every test now builds its own. Registration also outlived any workspace the operator closed, with no way to reset it. Both now live on AppState behind Arc>. The Arc rather than a plain Mutex field is deliberate: a command clones a cheap handle and carries it into spawn_blocking, so parsing a hundred thousand records never runs with the application state lock held. Inside the blocking work the lock is taken once for the whole file rather than per record, since a hundred thousand lock round trips would cost more than the parsing. apply_global becomes apply_registered and takes the registry. The parse paths, the live Windows query path, and every command thread it through explicitly. Co-Authored-By: Claude Opus 5 --- .../cmtraceopen-parser/src/event_query/mod.rs | 14 +- src-tauri/src/event_log/commands.rs | 75 ++++++-- src-tauri/src/event_log/live.rs | 34 +++- src-tauri/src/event_log/maps.rs | 61 +++---- src-tauri/src/event_log/parser.rs | 97 ++++++++-- src-tauri/src/event_log/provider_db.rs | 169 +++++++++--------- src-tauri/src/state/app_state.rs | 22 ++- src-tauri/tests/event_log_real_evtx.rs | 10 +- src/workspaces/event-log/EvtxFilterBar.tsx | 9 +- src/workspaces/event-log/EvtxTimeline.tsx | 8 +- 10 files changed, 323 insertions(+), 176 deletions(-) diff --git a/crates/cmtraceopen-parser/src/event_query/mod.rs b/crates/cmtraceopen-parser/src/event_query/mod.rs index fc3678792..48628290a 100644 --- a/crates/cmtraceopen-parser/src/event_query/mod.rs +++ b/crates/cmtraceopen-parser/src/event_query/mod.rs @@ -51,9 +51,17 @@ pub enum SelectorMode { #[serde(rename_all = "camelCase", tag = "kind")] pub enum EventIdSelector { /// A single ID. - Single { id: u32 }, + Single { + /// The Event ID, which identifies an event only together with its provider. + id: u32, + }, /// An inclusive `low..=high` range. - Range { low: u32, high: u32 }, + Range { + /// Lower bound, inclusive. Swapped with `high` if the two arrive reversed. + low: u32, + /// Upper bound, inclusive. + high: u32, + }, } impl EventIdSelector { @@ -106,7 +114,9 @@ pub enum TimeWindow { Last { milliseconds: u64 }, /// An explicit range. Bounds are ISO 8601 UTC, for example `2026-08-09T00:00:00.000Z`. Between { + /// Inclusive lower bound. `None` leaves the range open at the start. from: Option, + /// Inclusive upper bound. `None` leaves it open at the end. to: Option, }, } diff --git a/src-tauri/src/event_log/commands.rs b/src-tauri/src/event_log/commands.rs index a9832e8b6..4fd8492ba 100644 --- a/src-tauri/src/event_log/commands.rs +++ b/src-tauri/src/event_log/commands.rs @@ -6,6 +6,7 @@ use tauri::Emitter; use super::models::{EvtxChannelInfo, EvtxParseResult}; use super::parser; +use crate::state::app_state::AppState; #[cfg(target_os = "windows")] #[derive(Clone, Serialize)] @@ -16,8 +17,16 @@ struct EvtxQueryProgress { } #[tauri::command] -pub async fn evtx_parse_files(paths: Vec) -> Result { - tokio::task::spawn_blocking(move || parser::parse_evtx_files(&paths)) +pub async fn evtx_parse_files( + paths: Vec, + state: tauri::State<'_, AppState>, +) -> Result { + // Handles are cloned out before the blocking work starts. Parsing can run for a long time over + // a hundred thousand records, and holding a state lock across it would stall every other + // command. + let maps = state.event_maps.clone(); + let providers = state.provider_store.clone(); + tokio::task::spawn_blocking(move || parser::parse_evtx_files(&paths, &maps, &providers)) .await .map_err(|e| format!("Task join error: {}", e))? } @@ -42,12 +51,20 @@ pub async fn evtx_query_channels( max_events: Option, filter: Option, app: AppHandle, + state: tauri::State<'_, AppState>, ) -> Result { #[cfg(target_os = "windows")] { + // Cloned before the blocking work, so a long channel scan never runs with a state lock + // held. Channels are then read under one shared guard rather than locking per record. + let registry = state.event_maps.clone(); tokio::task::spawn_blocking(move || { use rayon::prelude::*; + let maps = registry + .read() + .map_err(|_| "map registry lock was poisoned".to_string())?; + // Absent means unfiltered, which keeps the query as "*" and preserves prior behaviour // for callers that have not adopted server-side filtering yet. let query_filter = filter.unwrap_or_default(); @@ -65,6 +82,7 @@ pub async fn evtx_query_channels( let outcome = super::live::query_channel_filtered_with_progress( channel, &query_filter, + &maps, max_events, |fetched, _| { let _ = app_ref.emit( @@ -134,7 +152,7 @@ pub async fn evtx_query_channels( } #[cfg(not(target_os = "windows"))] { - let _ = (channels, max_events, filter, app); + let _ = (channels, max_events, filter, app, state); Ok(EvtxParseResult { records: Vec::new(), channels: Vec::new(), @@ -168,24 +186,38 @@ pub async fn evtx_export_records( Ok(byte_count) } -/// Loads EvtxECmd `.map` files from `directory` into the process registry. +/// Loads EvtxECmd `.map` files from `directory` into the application's registry. /// /// Returns what loaded, what was superseded, and what failed, so an operator can see why an event /// type is not being mapped rather than being left guessing. #[tauri::command] pub async fn evtx_load_event_maps( directory: String, + state: tauri::State<'_, AppState>, ) -> Result { let path = std::path::PathBuf::from(&directory); - tokio::task::spawn_blocking(move || super::maps::load_global(&path)) - .await - .map_err(|error| format!("map load task failed: {error}"))? + let maps = state.event_maps.clone(); + tokio::task::spawn_blocking(move || { + // Read from disk first, then swap. Holding the write lock across the read would block + // every in-flight parse for as long as the directory takes to load. + let (registry, outcome) = super::maps::load_maps_from_dir(&path)?; + *maps + .write() + .map_err(|_| "map registry lock was poisoned".to_string())? = registry; + Ok(outcome) + }) + .await + .map_err(|error| format!("map load task failed: {error}"))? } /// Number of maps currently in effect. #[tauri::command] -pub async fn evtx_loaded_map_count() -> u64 { - super::maps::loaded_count() as u64 +pub async fn evtx_loaded_map_count(state: tauri::State<'_, AppState>) -> Result { + Ok(state + .event_maps + .read() + .map_err(|_| "map registry lock was poisoned".to_string())? + .len() as u64) } /// Registers every provider database in `directory` for description rendering. @@ -195,17 +227,32 @@ pub async fn evtx_loaded_map_count() -> u64 { #[tauri::command] pub async fn evtx_load_provider_databases( directory: String, + state: tauri::State<'_, AppState>, ) -> Result, String> { let path = std::path::PathBuf::from(&directory); - tokio::task::spawn_blocking(move || super::provider_db::load_directory(&path)) - .await - .map_err(|error| format!("provider database load task failed: {error}"))? + let providers = state.provider_store.clone(); + tokio::task::spawn_blocking( + move || -> Result, String> { + providers + .write() + .map_err(|_| "provider store lock was poisoned".to_string())? + .load_directory(&path) + }, + ) + .await + .map_err(|error| format!("provider database load task failed: {error}"))? } /// Provider databases currently registered. #[tauri::command] -pub async fn evtx_provider_databases() -> Vec { - super::provider_db::registered() +pub async fn evtx_provider_databases( + state: tauri::State<'_, AppState>, +) -> Result, String> { + Ok(state + .provider_store + .read() + .map_err(|_| "provider store lock was poisoned".to_string())? + .registered()) } /// Merges already-loaded log entries and event records into one chronological timeline. diff --git a/src-tauri/src/event_log/live.rs b/src-tauri/src/event_log/live.rs index 3953065ca..d2ad17765 100644 --- a/src-tauri/src/event_log/live.rs +++ b/src-tauri/src/event_log/live.rs @@ -7,6 +7,7 @@ use regex::Regex; use super::models::{ChannelSourceType, EvtxChannelInfo, EvtxField, EvtxLevel, EvtxRecord}; use super::{parse_timestamp_to_epoch_ms, sanitize_control_chars}; use cmtraceopen_parser::event_query::{build_query, EventQueryFilter}; +use cmtraceopen_parser::eventmap::MapRegistry; #[cfg(target_os = "windows")] use windows::core::{Error, HSTRING, PCWSTR}; @@ -125,8 +126,12 @@ pub fn enumerate_channels() -> Result, String> { /// /// Returns newest events first, capped at `max_events` (default 1000). #[cfg(target_os = "windows")] -pub fn query_channel(channel: &str, max_events: Option) -> Result, String> { - query_channel_with_progress(channel, max_events, |_, _| {}) +pub fn query_channel( + channel: &str, + maps: &MapRegistry, + max_events: Option, +) -> Result, String> { + query_channel_with_progress(channel, maps, max_events, |_, _| {}) } /// Queries a channel with server-side filtering. @@ -137,21 +142,24 @@ pub fn query_channel(channel: &str, max_events: Option) -> Result, ) -> Result, String> { - query_channel_inner(channel, filter, max_events, |_, _| {}) + query_channel_inner(channel, filter, maps, max_events, |_, _| {}) } /// Query with a progress callback: `on_progress(fetched_so_far, total_estimate)`. #[cfg(target_os = "windows")] pub fn query_channel_with_progress( channel: &str, + maps: &MapRegistry, max_events: Option, on_progress: impl Fn(usize, Option), ) -> Result, String> { query_channel_inner( channel, &EventQueryFilter::default(), + maps, max_events, on_progress, ) @@ -162,16 +170,18 @@ pub fn query_channel_with_progress( pub fn query_channel_filtered_with_progress( channel: &str, filter: &EventQueryFilter, + maps: &MapRegistry, max_events: Option, on_progress: impl Fn(usize, Option), ) -> Result, String> { - query_channel_inner(channel, filter, max_events, on_progress) + query_channel_inner(channel, filter, maps, max_events, on_progress) } #[cfg(target_os = "windows")] fn query_channel_inner( channel: &str, filter: &EventQueryFilter, + maps: &MapRegistry, max_events: Option, on_progress: impl Fn(usize, Option), ) -> Result, String> { @@ -246,7 +256,9 @@ fn query_channel_inner( .flatten() }); - if let Some(record) = parse_xml_to_record(&xml, channel, rendered_message.as_deref()) { + if let Some(record) = + parse_xml_to_record(&xml, channel, maps, rendered_message.as_deref()) + { records.push(record); // Report progress every 100 records if records.len() % 100 == 0 { @@ -279,6 +291,7 @@ pub fn enumerate_channels() -> Result, String> { #[cfg(not(target_os = "windows"))] pub fn query_channel_with_progress( _channel: &str, + _maps: &MapRegistry, _max_events: Option, _on_progress: impl Fn(usize, Option), ) -> Result, String> { @@ -286,7 +299,11 @@ pub fn query_channel_with_progress( } #[cfg(not(target_os = "windows"))] -pub fn query_channel(_channel: &str, _max_events: Option) -> Result, String> { +pub fn query_channel( + _channel: &str, + _maps: &MapRegistry, + _max_events: Option, +) -> Result, String> { Err("Live event log queries are only available on Windows.".to_string()) } @@ -294,6 +311,7 @@ pub fn query_channel(_channel: &str, _max_events: Option) -> Result, ) -> Result, String> { Err("Live event log queries are only available on Windows.".to_string()) @@ -303,6 +321,7 @@ pub fn query_channel_filtered( pub fn query_channel_filtered_with_progress( _channel: &str, _filter: &EventQueryFilter, + _maps: &MapRegistry, _max_events: Option, _on_progress: impl Fn(usize, Option), ) -> Result, String> { @@ -401,6 +420,7 @@ fn format_event_message( fn parse_xml_to_record( xml: &str, channel: &str, + maps: &MapRegistry, rendered_message: Option<&str>, ) -> Option { let event_id_str = extract_xml_text(xml, "EventID").unwrap_or_default(); @@ -453,7 +473,7 @@ fn parse_xml_to_record( .unwrap_or_default(); let mapped = parsed .as_ref() - .map(|root| super::maps::apply_global(channel, &provider, event_id, root)) + .map(|root| super::maps::apply_registered(maps, channel, &provider, event_id, root)) .unwrap_or_default(); Some(EvtxRecord { diff --git a/src-tauri/src/event_log/maps.rs b/src-tauri/src/event_log/maps.rs index c95887529..fc9f42092 100644 --- a/src-tauri/src/event_log/maps.rs +++ b/src-tauri/src/event_log/maps.rs @@ -18,7 +18,6 @@ use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; -use std::sync::{OnceLock, RwLock}; use cmtraceopen_parser::eventmap::{apply_map, EventMap, EventNode, MapProperty, MapRegistry}; use serde::{Deserialize, Serialize}; @@ -414,18 +413,6 @@ Maps: } } -// ── Process-wide registry ─────────────────────────────────────────────────── - -/// The maps in effect for this process. -/// -/// Held globally rather than threaded through every call site because both the live and file -/// record paths already parse the event once and applying maps there avoids a second parse. An -/// empty registry is the normal state until maps are loaded, and it simply yields no columns. -fn global() -> &'static RwLock { - static REGISTRY: OnceLock> = OnceLock::new(); - REGISTRY.get_or_init(|| RwLock::new(MapRegistry::new())) -} - /// One normalized column produced by a map. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] @@ -449,33 +436,21 @@ fn property_name(property: &MapProperty) -> String { } } -/// Replaces the process registry with the maps in `directory`. -pub fn load_global(directory: &Path) -> Result { - let (registry, outcome) = load_maps_from_dir(directory)?; - *global() - .write() - .map_err(|_| "map registry lock was poisoned".to_string())? = registry; - Ok(outcome) -} - -/// Number of maps currently loaded. -pub fn loaded_count() -> usize { - global().read().map(|registry| registry.len()).unwrap_or(0) -} - /// Applies the registered map for this event, if one exists. /// /// Returns an empty vector when no map matches, which is the common case: the upstream corpus /// covers a few hundred event types out of many thousands. -pub fn apply_global( +/// +/// The registry is passed in rather than reached for. Both record paths already parse the event +/// once and apply maps there, so whoever owns the registry hands it down; there is no process +/// global to make two tests, or two windows, share one set of maps. +pub fn apply_registered( + registry: &MapRegistry, channel: &str, provider: &str, event_id: u32, event: &EventNode, ) -> Vec { - let Ok(registry) = global().read() else { - return Vec::new(); - }; let Some(map) = registry.find(channel, provider, event_id) else { return Vec::new(); }; @@ -499,28 +474,37 @@ mod global_tests { fn an_unloaded_registry_yields_no_columns_rather_than_failing() { let event = EventNode::new("Event"); // Whatever other tests have loaded, an event with no matching map must map to nothing. - assert!(apply_global("No-Such-Channel", "No-Such-Provider", 1, &event).is_empty()); + assert!(apply_registered( + &MapRegistry::new(), + "No-Such-Channel", + "No-Such-Provider", + 1, + &event + ) + .is_empty()); } #[test] fn a_loaded_map_produces_columns_for_a_matching_event_end_to_end() { - // Proves the whole chain: YAML on disk, into the process registry, applied to XML parsed - // by the host adapter, out as columns the UI can render. + // Proves the whole chain: YAML on disk, into a registry, applied to XML parsed by the + // host adapter, out as columns the UI can render. The registry is local to this test, so + // it cannot be disturbed by another test loading a different set on a parallel thread. let dir = std::env::temp_dir().join("cmtraceopen-maps-global-e2e"); let _ = fs::remove_dir_all(&dir); fs::create_dir_all(&dir).expect("temp dir"); fs::write(dir.join("shell-core-9701.map"), SHELL_CORE_9701).expect("write map"); - let outcome = load_global(&dir).expect("loads"); + let (registry, outcome) = load_maps_from_dir(&dir).expect("loads"); assert_eq!(outcome.loaded_count(), 1); - assert!(loaded_count() >= 1); + assert_eq!(registry.len(), 1); let event = crate::event_log::event_node::parse_event_xml( "RunOnceEx started", ) .expect("parses"); - let columns = apply_global( + let columns = apply_registered( + ®istry, "Microsoft-Windows-Shell-Core/Operational", "Microsoft-Windows-Shell-Core", 9701, @@ -532,7 +516,8 @@ mod global_tests { assert!(columns[0].complete); // A different event id on the same channel has no map and must map to nothing. - assert!(apply_global( + assert!(apply_registered( + ®istry, "Microsoft-Windows-Shell-Core/Operational", "Microsoft-Windows-Shell-Core", 9702, diff --git a/src-tauri/src/event_log/parser.rs b/src-tauri/src/event_log/parser.rs index 84d1d6176..a918d0a64 100644 --- a/src-tauri/src/event_log/parser.rs +++ b/src-tauri/src/event_log/parser.rs @@ -1,7 +1,11 @@ use std::path::Path; +use std::sync::RwLock; +use cmtraceopen_parser::eventmap::MapRegistry; use evtx::EvtxParser; +use super::provider_db::ProviderStore; + use super::models::{ ChannelSourceType, EvtxChannelInfo, EvtxField, EvtxLevel, EvtxParseResult, EvtxRecord, }; @@ -11,7 +15,15 @@ use super::{parse_timestamp_to_epoch_ms, sanitize_control_chars}; const MAX_ENTRIES_PER_FILE: usize = 100_000; /// Parse one or more .evtx files and return a unified result. -pub fn parse_evtx_files(paths: &[String]) -> Result { +/// +/// The map registry and provider store are passed in rather than reached for. They belong to the +/// application state, so the caller decides which set is in effect; that is also what lets a test +/// use its own without another test on a parallel thread replacing it. +pub fn parse_evtx_files( + paths: &[String], + maps: &RwLock, + providers: &RwLock, +) -> Result { let mut all_records = Vec::new(); let mut channels = Vec::new(); let mut parse_errors = 0u32; @@ -19,7 +31,7 @@ pub fn parse_evtx_files(paths: &[String]) -> Result { for path_str in paths { let path = Path::new(path_str); - match parse_single_file(path) { + match parse_single_file(path, maps, providers) { Ok(file) => { let records = file.records; parse_errors += file.parse_errors; @@ -108,7 +120,11 @@ struct ParsedFile { /// record whose XML will not parse, and a file so large it was truncated are all cases where the /// view is incomplete, and a view that is silently incomplete is worse than one that is empty: /// the absent events look like evidence that the thing being investigated did not happen. -fn parse_single_file(path: &Path) -> Result { +fn parse_single_file( + path: &Path, + maps: &RwLock, + providers: &RwLock, +) -> Result { let mut parser = EvtxParser::from_path(path) .map_err(|e| format!("Failed to open EVTX file {}: {}", path.display(), e))?; @@ -122,6 +138,15 @@ fn parse_single_file(path: &Path) -> Result { let mut messages = Vec::new(); let mut truncated = false; + // Locked once for the whole file rather than per record. A hundred thousand lock round trips + // would cost more than the parsing does. + let maps = maps + .read() + .map_err(|_| "map registry lock was poisoned".to_string())?; + let mut providers = providers + .write() + .map_err(|_| "provider store lock was poisoned".to_string())?; + // XML rather than JSON. The JSON projection cannot be re-parsed into an event tree, which is // what the map engine, the System block, and the XML export all consume; reading XML here is // what makes those work on an opened file at all. @@ -198,11 +223,11 @@ fn parse_single_file(path: &Path) -> Result { // A provider database, when one is loaded, turns raw field values into the sentence the // provider intended. Without it the file path can only summarise EventData, which is what // every other cross-platform reader shows and why they are hard to read. - let message = describe_event(&provider, event_id, &insertions) + let message = describe_event(&mut providers, &provider, event_id, &insertions) .or(payload) .unwrap_or_else(|| build_message(&fields)); - let mapped = super::maps::apply_global(&channel, &provider, event_id, &parsed); + let mapped = super::maps::apply_registered(&maps, &channel, &provider, event_id, &parsed); records.push(EvtxRecord { id: 0, // Will be reassigned after sorting event_record_id, @@ -333,8 +358,13 @@ struct EventFields { /// A partially rendered description is rejected rather than shown. If the template references /// insertions the event did not supply, the metadata and the event disagree, and a sentence with /// `%4` embedded in it is less honest than the field summary it would replace. -fn describe_event(provider: &str, event_id: u32, insertions: &[String]) -> Option { - let metadata = super::provider_db::provider(provider)?; +fn describe_event( + store: &mut ProviderStore, + provider: &str, + event_id: u32, + insertions: &[String], +) -> Option { + let metadata = store.provider(provider)?; let event = metadata.event(event_id, None)?; let template = event.description.as_deref()?; @@ -371,6 +401,17 @@ mod tests { assert_eq!(EvtxLevel::from_level_value(255), EvtxLevel::Information); } + /// Empty registries, for tests that only care about parsing. + /// + /// Each test gets its own, so nothing here can be perturbed by another test on a parallel + /// thread loading a different set. + fn empty_state() -> (RwLock, RwLock) { + ( + RwLock::new(MapRegistry::new()), + RwLock::new(ProviderStore::default()), + ) + } + fn parse(xml: &str) -> cmtraceopen_parser::eventmap::EventNode { super::super::event_node::parse_event_xml(xml).expect("well formed") } @@ -387,7 +428,9 @@ mod tests { fn a_file_that_cannot_be_opened_is_named_in_the_result() { // A count with no file name and no reason leaves an operator with a number and no next // step. The message is what makes a missing log actionable. - let result = parse_evtx_files(&["/no/such/file.evtx".to_string()]).expect("returns"); + let (maps, providers) = empty_state(); + let result = parse_evtx_files(&["/no/such/file.evtx".to_string()], &maps, &providers) + .expect("returns"); assert_eq!(result.parse_errors, 1); assert_eq!(result.error_messages.len(), 1); assert!( @@ -401,7 +444,8 @@ mod tests { #[test] fn a_clean_parse_reports_nothing() { // The messages are a gap report, so an empty run must not manufacture one. - let result = parse_evtx_files(&[]).expect("returns"); + let (maps, providers) = empty_state(); + let result = parse_evtx_files(&[], &maps, &providers).expect("returns"); assert_eq!(result.parse_errors, 0); assert!(result.error_messages.is_empty()); assert_eq!(result.total_records, 0); @@ -623,31 +667,47 @@ mod description_tests { .collect() } + /// A store with nothing registered, which is the state until an operator loads a database. + fn empty_store() -> ProviderStore { + ProviderStore::default() + } + + /// A store with the databases beside `CMTRACEOPEN_PROVIDER_DB` registered. + /// + /// Built per test. A shared one would let these interfere with each other, since registering a + /// directory replaces whatever was there. + fn loaded_store() -> ProviderStore { + let path = std::env::var("CMTRACEOPEN_PROVIDER_DB").expect("database path"); + let directory = std::path::Path::new(&path) + .parent() + .expect("database has a parent directory"); + let mut store = ProviderStore::default(); + store.load_directory(directory).expect("databases load"); + store + } + #[test] fn with_no_database_loaded_it_falls_back_to_the_field_summary() { // The common case until an operator loads metadata. Must not fail or blank the message. let data = insertions(&[("HRESULT", "0x80180005")]); - assert!(describe_event("Nobody-Has-This-Provider", 1, &data).is_none()); + assert!(describe_event(&mut empty_store(), "Nobody-Has-This-Provider", 1, &data).is_none()); } #[test] fn an_unknown_event_id_falls_back_rather_than_inventing_a_description() { let data = insertions(&[("X", "1")]); - assert!(describe_event("Still-Not-Loaded", 999_999, &data).is_none()); + assert!(describe_event(&mut empty_store(), "Still-Not-Loaded", 999_999, &data).is_none()); } #[test] #[ignore = "requires a real provider database via CMTRACEOPEN_PROVIDER_DB"] fn a_loaded_database_renders_a_real_provider_description() { // The whole chain: SQLite on disk, gzip payload, provider metadata, insertion rendering. - let path = std::env::var("CMTRACEOPEN_PROVIDER_DB").expect("database path"); - let directory = std::path::Path::new(&path) - .parent() - .expect("database has a parent directory"); - super::super::provider_db::load_directory(directory).expect("databases load"); + let mut store = loaded_store(); let data = insertions(&[("HRESULT", "0x80180005")]); let described = describe_event( + &mut store, "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider", 2, &data, @@ -666,12 +726,11 @@ mod description_tests { #[test] #[ignore = "requires a real provider database via CMTRACEOPEN_PROVIDER_DB"] fn an_event_the_database_does_not_cover_still_falls_back() { - let path = std::env::var("CMTRACEOPEN_PROVIDER_DB").expect("database path"); - let directory = std::path::Path::new(&path).parent().expect("parent"); - super::super::provider_db::load_directory(directory).expect("databases load"); + let mut store = loaded_store(); // A provider that genuinely is not in a Windows capture. assert!(describe_event( + &mut store, "Definitely-Not-A-Real-Provider", 1, &insertions(&[("a", "b")]) diff --git a/src-tauri/src/event_log/provider_db.rs b/src-tauri/src/event_log/provider_db.rs index b14f8e5d1..0d038395d 100644 --- a/src-tauri/src/event_log/provider_db.rs +++ b/src-tauri/src/event_log/provider_db.rs @@ -22,7 +22,6 @@ use std::collections::HashMap; use std::io::Read; use std::path::{Path, PathBuf}; -use std::sync::{OnceLock, RwLock}; use cmtraceopen_parser::provider::ProviderMetadata; use flate2::read::GzDecoder; @@ -186,10 +185,16 @@ impl ProviderDb { } } -// ── Process-wide store ────────────────────────────────────────────────────── +// ── Store ─────────────────────────────────────────────────────────────────── +/// Registered provider databases and the metadata read from them. +/// +/// Owned by `AppState` rather than held as a process global. A global made every test share one +/// set: `load_directory` clears and replaces it, and cargo runs tests on parallel threads in one +/// process, so two tests registering databases would interfere. It also meant the registration +/// outlived any workspace the operator closed, with no way to reset it. #[derive(Default)] -struct ProviderStore { +pub struct ProviderStore { databases: Vec, /// Lowercased provider name to metadata, including negative results so a provider absent from /// every database is not looked up again for every event that mentions it. @@ -197,101 +202,84 @@ struct ProviderStore { info: Vec, } -fn store() -> &'static RwLock { - static STORE: OnceLock> = OnceLock::new(); - STORE.get_or_init(|| RwLock::new(ProviderStore::default())) -} - -/// Registers every `.db` in `directory`, replacing any previously registered set. -pub fn load_directory(directory: &Path) -> Result, String> { - let entries = std::fs::read_dir(directory).map_err(|error| { - format!( - "cannot read provider database directory {}: {error}", - directory.display() - ) - })?; - - let mut databases = Vec::new(); - let mut info = Vec::new(); - let mut failures = Vec::new(); - - let mut paths: Vec = entries - .filter_map(Result::ok) - .map(|entry| entry.path()) - .filter(|path| { - path.is_file() - && path - .extension() - .and_then(|extension| extension.to_str()) - .is_some_and(|extension| extension.eq_ignore_ascii_case("db")) - }) - .collect(); - paths.sort(); - - for path in paths { - match ProviderDb::open(&path) { - Ok(database) => { - info.push(database.info().clone()); - databases.push(path); +impl ProviderStore { + /// Registers every `.db` in `directory`, replacing any previously registered set. + pub fn load_directory(&mut self, directory: &Path) -> Result, String> { + let entries = std::fs::read_dir(directory).map_err(|error| { + format!( + "cannot read provider database directory {}: {error}", + directory.display() + ) + })?; + + let mut databases = Vec::new(); + let mut info = Vec::new(); + let mut failures = Vec::new(); + + let mut paths: Vec = entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| { + path.is_file() + && path + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("db")) + }) + .collect(); + paths.sort(); + + for path in paths { + match ProviderDb::open(&path) { + Ok(database) => { + info.push(database.info().clone()); + databases.push(path); + } + // A file that is not a provider database is reported, not fatal: the directory is + // user-supplied and may hold anything. + Err(reason) => failures.push(reason), } - // A file that is not a provider database is reported, not fatal: the directory is - // user-supplied and may hold anything. - Err(reason) => failures.push(reason), } - } - { - let mut guard = store() - .write() - .map_err(|_| "provider store lock was poisoned".to_string())?; - guard.databases = databases; - guard.info = info.clone(); - guard.cache.clear(); - } + self.databases = databases; + self.info = info.clone(); + self.cache.clear(); - if info.is_empty() && !failures.is_empty() { - return Err(failures.join("; ")); + if info.is_empty() && !failures.is_empty() { + return Err(failures.join("; ")); + } + Ok(info) } - Ok(info) -} -/// Metadata for `provider_name`, consulting registered databases in order and caching the result. -pub fn provider(provider_name: &str) -> Option { - let key = provider_name.to_ascii_lowercase(); - - if let Ok(guard) = store().read() { - if let Some(cached) = guard.cache.get(&key) { + /// Metadata for `provider_name`, consulting registered databases in order and caching it. + /// + /// Takes `&mut self` because a lookup populates the cache, including the negative result. That + /// is what stops a provider absent from every database being searched again for every event + /// that names it. + pub fn provider(&mut self, provider_name: &str) -> Option { + let key = provider_name.to_ascii_lowercase(); + if let Some(cached) = self.cache.get(&key) { return cached.clone(); } - } - let paths = match store().read() { - Ok(guard) => guard.databases.clone(), - Err(_) => return None, - }; - - let mut found = None; - for path in paths { - if let Ok(database) = ProviderDb::open(&path) { - if let Ok(Some(metadata)) = database.provider(provider_name) { - found = Some(metadata); - break; + let mut found = None; + for path in &self.databases { + if let Ok(database) = ProviderDb::open(path) { + if let Ok(Some(metadata)) = database.provider(provider_name) { + found = Some(metadata); + break; + } } } - } - if let Ok(mut guard) = store().write() { - guard.cache.insert(key, found.clone()); + self.cache.insert(key, found.clone()); + found } - found -} -/// Summary of every registered database. -pub fn registered() -> Vec { - store() - .read() - .map(|guard| guard.info.clone()) - .unwrap_or_default() + /// Summary of every registered database. + pub fn registered(&self) -> Vec { + self.info.clone() + } } #[cfg(test)] @@ -479,12 +467,15 @@ mod tests { build_db(&dir.join("b.db"), &[("B", 26200, EVENTS)]); std::fs::write(dir.join("notes.txt"), b"ignore me").expect("write"); - let info = load_directory(&dir).expect("loads"); + // A store local to this test. The old process global meant a parallel test registering a + // different directory replaced this one's set mid-run. + let mut store = ProviderStore::default(); + let info = store.load_directory(&dir).expect("loads"); assert_eq!(info.len(), 2); - assert!(provider("A").is_some()); - assert!(provider("B").is_some()); - assert!(provider("Nobody").is_none()); - assert_eq!(registered().len(), 2); + assert!(store.provider("A").is_some()); + assert!(store.provider("B").is_some()); + assert!(store.provider("Nobody").is_none()); + assert_eq!(store.registered().len(), 2); } } diff --git a/src-tauri/src/state/app_state.rs b/src-tauri/src/state/app_state.rs index a87cac1a8..ad7a6caa9 100644 --- a/src-tauri/src/state/app_state.rs +++ b/src-tauri/src/state/app_state.rs @@ -1,11 +1,15 @@ use std::collections::HashMap; use std::path::PathBuf; -#[cfg(feature = "esp-diagnostics")] +#[cfg(any(feature = "esp-diagnostics", feature = "event-log"))] use std::sync::Arc; use std::sync::Mutex; +#[cfg(feature = "event-log")] +use std::sync::RwLock; #[cfg(feature = "esp-diagnostics")] use crate::esp::session::{EspSessionError, EspSessionManager}; +#[cfg(feature = "event-log")] +use crate::event_log::provider_db::ProviderStore; use crate::parser::ResolvedParser; #[cfg(feature = "sccm-diagnostics")] use crate::sccm::collector::SccmAdvancedCapabilityStore; @@ -46,6 +50,18 @@ pub struct AppState { /// its worker and AppHandle-backed event sink cannot outlive the runtime. #[cfg(feature = "esp-diagnostics")] esp_session_manager: Mutex>>, + /// Event maps loaded from disk, applied while rendering event rows. + /// + /// Behind an `Arc>` rather than a `Mutex` on the state itself so a command can take + /// a cheap handle and carry it into `spawn_blocking`. Parsing a hundred thousand records is + /// exactly the blocking work that must not run while the application state lock is held. + #[cfg(feature = "event-log")] + pub event_maps: Arc>, + /// Provider metadata databases, read to render an event's own description. + /// + /// Held the same way and for the same reason as [`event_maps`](Self::event_maps). + #[cfg(feature = "event-log")] + pub provider_store: Arc>, } impl AppState { @@ -69,6 +85,10 @@ impl AppState { sccm_advanced_capabilities: Mutex::new(SccmAdvancedCapabilityStore::default()), #[cfg(feature = "esp-diagnostics")] esp_session_manager: Mutex::new(None), + #[cfg(feature = "event-log")] + event_maps: Arc::new(RwLock::new(Default::default())), + #[cfg(feature = "event-log")] + provider_store: Arc::new(RwLock::new(ProviderStore::default())), } } diff --git a/src-tauri/tests/event_log_real_evtx.rs b/src-tauri/tests/event_log_real_evtx.rs index a41920730..509bd5869 100644 --- a/src-tauri/tests/event_log_real_evtx.rs +++ b/src-tauri/tests/event_log_real_evtx.rs @@ -22,6 +22,9 @@ use std::path::PathBuf; use app_lib::event_log::export::{export_records, ExportFormat}; use app_lib::event_log::models::EvtxParseResult; use app_lib::event_log::parser::parse_evtx_files; +use app_lib::event_log::provider_db::ProviderStore; +use cmtraceopen_parser::eventmap::MapRegistry; +use std::sync::RwLock; fn fixture() -> Option { let Some(raw) = std::env::var_os("CMTRACE_EVTX_FIXTURE") else { @@ -45,7 +48,12 @@ fn fixture() -> Option { fn parsed() -> Option { let path = fixture()?; - let result = parse_evtx_files(&[path.to_string_lossy().into_owned()]).expect("the file parses"); + // Registries local to this call. The parse path takes them explicitly, so nothing here can be + // perturbed by another test loading a different set on a parallel thread. + let maps = RwLock::new(MapRegistry::new()); + let providers = RwLock::new(ProviderStore::default()); + let result = parse_evtx_files(&[path.to_string_lossy().into_owned()], &maps, &providers) + .expect("the file parses"); assert!( !result.records.is_empty(), "fixture produced no records at all" diff --git a/src/workspaces/event-log/EvtxFilterBar.tsx b/src/workspaces/event-log/EvtxFilterBar.tsx index 0c579a50b..cf71eda63 100644 --- a/src/workspaces/event-log/EvtxFilterBar.tsx +++ b/src/workspaces/event-log/EvtxFilterBar.tsx @@ -152,11 +152,18 @@ export function EvtxFilterBar() { }); if (!destination) return; setExportState("Exporting..."); - const bytes = await invoke("evtx_export_records", { + const bytes = await invoke("evtx_export_records", { records, format: format.value, destination, }); + // The IPC boundary is typed by assertion, not by the compiler. A malformed reply would + // otherwise render as "Exported ... (NaN KB)", which still reads as success, and an operator + // would believe a file was written. + if (typeof bytes !== "number" || !Number.isFinite(bytes) || bytes < 0) { + setExportState("Export failed: the writer did not report how much it wrote"); + return; + } setExportState(`Exported ${records.length} events (${Math.round(bytes / 1024)} KB)`); } catch (error) { setExportState( diff --git a/src/workspaces/event-log/EvtxTimeline.tsx b/src/workspaces/event-log/EvtxTimeline.tsx index 4428ee281..ca6671f08 100644 --- a/src/workspaces/event-log/EvtxTimeline.tsx +++ b/src/workspaces/event-log/EvtxTimeline.tsx @@ -265,11 +265,11 @@ export function EvtxTimeline() { tabIndex={-1} onClick={() => toggleGroup(row.key)} style={{ - position: "absolute", - top: 0, - left: 0, + // Normal flow, matching the record rows. The wrapper above is already + // translated to the first visible row's offset and its children stack inside + // it, so positioning a header absolutely applied that offset a second time and + // took it out of flow, letting the rows beneath slide up into its place. width: "100%", - transform: `translateY(${virtualRow.start}px)`, display: "flex", alignItems: "center", gap: "6px", From 033e8f560d661cc2478535fcdcb955f1003b16e4 Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 9 Aug 2026 22:39:15 -0400 Subject: [PATCH 47/85] fix(event-log): pass the registry in the live service tests The Windows-only test module still called the query functions at their old arity, which only that platform's build could see. Co-Authored-By: Claude Opus 5 --- src-tauri/src/event_log/live.rs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src-tauri/src/event_log/live.rs b/src-tauri/src/event_log/live.rs index d2ad17765..104f7d483 100644 --- a/src-tauri/src/event_log/live.rs +++ b/src-tauri/src/event_log/live.rs @@ -636,7 +636,8 @@ mod tests { let has_app = channels.iter().any(|c| c.name == "Application"); println!("Has Application channel: {has_app}"); - let records = query_channel("Application", Some(3)).expect("query should work"); + let records = + query_channel("Application", &MapRegistry::new(), Some(3)).expect("query should work"); println!("Application records: {}", records.len()); for (i, r) in records.iter().enumerate() { println!("--- Record {i} ---"); @@ -671,10 +672,16 @@ mod live_service_tests { const CHANNEL: &str = "Application"; + /// A registry local to the call, replacing what used to be an implicit process global. These + /// tests are about the query reaching the service, not about mapping. + fn no_maps() -> MapRegistry { + MapRegistry::new() + } + #[test] #[ignore = "requires a live Windows Event Log service with events"] fn an_unfiltered_query_returns_records() { - let records = query_channel(CHANNEL, Some(50)).expect("query succeeds"); + let records = query_channel(CHANNEL, &no_maps(), Some(50)).expect("query succeeds"); assert!( !records.is_empty(), "Application channel should have events" @@ -701,6 +708,8 @@ mod live_service_tests { }), ..Default::default() }, + &no_maps(), + &no_maps(), None, ) .expect("1 hour query succeeds"); @@ -713,6 +722,8 @@ mod live_service_tests { }), ..Default::default() }, + &no_maps(), + &no_maps(), None, ) .expect("30 day query succeeds"); @@ -736,6 +747,7 @@ mod live_service_tests { levels: vec![2], ..Default::default() }, + &no_maps(), Some(200), ) .expect("level query succeeds"); @@ -762,6 +774,7 @@ mod live_service_tests { }], ..Default::default() }, + &no_maps(), Some(50), ) .expect("query succeeds"); @@ -776,8 +789,9 @@ mod live_service_tests { #[test] #[ignore = "requires a live Windows Event Log service with events"] fn system_fields_are_populated_from_real_events() { - let records = query_channel_filtered(CHANNEL, &EventQueryFilter::default(), Some(200)) - .expect("query succeeds"); + let records = + query_channel_filtered(CHANNEL, &EventQueryFilter::default(), &no_maps(), Some(200)) + .expect("query succeeds"); assert!(!records.is_empty()); assert!( From d6ffbbd4887d12a99438441e265241ec293b9c21 Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 9 Aug 2026 22:42:20 -0400 Subject: [PATCH 48/85] fix(event-log): remove a duplicated argument in two service tests Co-Authored-By: Claude Opus 5 --- src-tauri/src/event_log/live.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src-tauri/src/event_log/live.rs b/src-tauri/src/event_log/live.rs index 104f7d483..7978de14e 100644 --- a/src-tauri/src/event_log/live.rs +++ b/src-tauri/src/event_log/live.rs @@ -709,7 +709,6 @@ mod live_service_tests { ..Default::default() }, &no_maps(), - &no_maps(), None, ) .expect("1 hour query succeeds"); @@ -723,7 +722,6 @@ mod live_service_tests { ..Default::default() }, &no_maps(), - &no_maps(), None, ) .expect("30 day query succeeds"); From 8b3d9509ce3c5dcc6eb69257beccca559792a2ca Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 9 Aug 2026 23:42:46 -0400 Subject: [PATCH 49/85] fix(event-log): stop an oversized exclusion filter silently returning nothing Eighteen review findings. The first was a genuinely bad interaction between two things I had already added. An exclusion list is never split, because "not (a or b)" spread across unioned nodes means "not a or not b", which matches nearly everything. So a filter excluding more than about twenty event IDs emitted one node above the 24-expression cliff measured earlier on this branch, and the service rejected it. Production sets EvtQueryTolerateQueryErrors, which turns that refusal into a channel reporting no events. The filter looked like it worked and returned nothing. Expressed with instead, which the service subtracts from the selection rather than unioning, so chunking is safe. Measured on Windows 11 build 26200: the 30-term "!=" chain the builder used to emit is rejected with ERROR_EVT_INVALID_QUERY; the chunked suppression form is accepted; and on a list small enough for both, the two return identical counts (15,330 each). The builder's own output for 30 exclusions now returns 14,702 where it previously returned nothing at all. Also fixed: - provider metadata was deep-cloned on every cache hit, once per record, so opening one file could clone a provider carrying fifty event descriptions a hundred thousand times. Cached behind an Arc, and the cache moved behind its own Mutex so a lookup needs only a read guard on the store rather than blocking every reader for the length of a file. - the provider loader held its write lock across the whole directory scan, and discarded both enumeration errors and per-database failures whenever at least one database loaded, leaving an operator with partial coverage and no reason. It now scans into a fresh store and swaps, matching the map loader. - MapProperty, PathError and UnplacedReason are non_exhaustive. All three enumerate a category that will grow, and the attribute is only free before the first release that exposes them. - group headers were pointer-only, so a keyboard user could not expand or collapse a group at all - a width set for the Description column was ignored, so resizing it did nothing - the range-costing test passed through a disjunct that was true for another reason and would still have passed with the split logic deleted; it now asserts ten ranges sit at exactly the budget and eleven split - the corpus test inferred parse coverage from apply_map, which skips bindings whose placeholder is absent from the template, so a malformed expression on one was never parsed. Every expression is parsed directly now. - the module doc still claimed a bound of ten per node, and TimeWindow::Last carried an undocumented public field Co-Authored-By: Claude Opus 5 --- .../cmtraceopen-parser/src/event_query/mod.rs | 179 +++++++++++++++--- .../cmtraceopen-parser/src/eventmap/model.rs | 4 + .../cmtraceopen-parser/src/eventmap/path.rs | 4 + .../src/unified_timeline/mod.rs | 4 + .../tests/eventmap_corpus.rs | 32 +++- src-tauri/src/event_log/commands.rs | 12 +- src-tauri/src/event_log/maps.rs | 4 + src-tauri/src/event_log/parser.rs | 22 ++- src-tauri/src/event_log/provider_db.rs | 69 +++++-- src/workspaces/event-log/EvtxTimeline.tsx | 11 +- src/workspaces/event-log/EvtxTimelineRow.tsx | 7 +- 11 files changed, 275 insertions(+), 73 deletions(-) diff --git a/crates/cmtraceopen-parser/src/event_query/mod.rs b/crates/cmtraceopen-parser/src/event_query/mod.rs index 48628290a..3e03dfc6d 100644 --- a/crates/cmtraceopen-parser/src/event_query/mod.rs +++ b/crates/cmtraceopen-parser/src/event_query/mod.rs @@ -13,7 +13,8 @@ //! //! - **Expression count is capped.** A query with too many `or` terms is rejected outright, so //! large Event ID sets are split across several `` minus chunked `` nodes. +/// +/// The selection carries every predicate other than the Event IDs; the suppressions carry the IDs +/// as ordinary equalities. The service removes each suppression's matches from the selection, and +/// several suppressions remove the union of their matches, which is exactly what excluding a list +/// means. +fn build_suppressed_query(filter: &EventQueryFilter) -> Result { + let selection = select_body(filter, &[])?; + + let mut query = String::from(""); + + // Suppressions are written as inclusions of what to remove, so each selector costs what it + // would cost in an include list. + for chunk in chunk_by_expression_budget(&filter.event_ids, 0) { + let predicates: Vec = chunk + .iter() + .map(|selector| selector.predicate(SelectorMode::Include)) + .collect(); + let body = format!("*[System[({})]]", join_or(&predicates)); + let _ = write!(query, "{}", escape_for_xml(&body)); + } + + query.push_str(""); + Ok(query) +} + #[cfg(test)] mod tests { use super::*; @@ -609,11 +653,11 @@ mod tests { } #[test] - fn an_exclusion_list_is_never_split_because_union_would_invert_it() { - // "not (a or b)" spread across unioned nodes becomes "not a or not b", which matches - // almost everything. Excludes stay in one node even when large. + fn a_small_exclusion_list_stays_one_expression() { + // Within budget it is written as "!=" joined by "and". The subset has no negation, so that + // is the only way to say it in a single node. let mut f = filter(); - f.event_ids = (1..=45).map(|id| EventIdSelector::Single { id }).collect(); + f.event_ids = (1..=5).map(|id| EventIdSelector::Single { id }).collect(); f.event_id_mode = SelectorMode::Exclude; let query = build_query(&f).expect("builds"); @@ -625,6 +669,67 @@ mod tests { assert!(query.contains("EventID!=1 and EventID!=2"), "{query}"); } + #[test] + fn an_oversized_exclusion_list_becomes_suppressions_rather_than_a_rejected_query() { + // Never unioned nodes: "not (a or b)" spread that way means "not a or not b", + // which matches nearly everything. is subtracted from the selection instead, so + // chunking it is safe. + let mut f = filter(); + f.event_ids = (1..=45).map(|id| EventIdSelector::Single { id }).collect(); + f.event_id_mode = SelectorMode::Exclude; + let query = build_query(&f).expect("builds"); + + assert!(query.starts_with(""), "{query}"); + assert_eq!( + query.matches(", or + // the query would return far more than the filter asked for. + let mut f = filter(); + f.levels = vec![1, 2]; + f.event_ids = (1..=45).map(|id| EventIdSelector::Single { id }).collect(); + f.event_id_mode = SelectorMode::Exclude; + let query = build_query(&f).expect("builds"); + + let selection = query + .split("").next()) + .expect("a selection"); + assert!(selection.contains("Level=1 or Level=2"), "{selection}"); + assert!(!selection.contains("EventID"), "{selection}"); + } + #[test] fn a_set_that_fits_the_expression_budget_is_not_split() { let mut f = filter(); @@ -976,11 +1081,12 @@ mod expression_budget_tests { } #[test] - fn ten_ranges_split_even_though_ten_singles_would_not() { - // Ten selectors either way. Counting selectors would emit one node of 20 expressions for - // the ranges, which is the documented ceiling before a structured query is required. - let ranges = EventQueryFilter { - event_ids: (0..10) + fn ranges_are_costed_at_two_expressions_each() { + // Ten selectors either way, but a range is two comparisons. The concrete shape is asserted + // rather than "split or exactly twenty": the disjunction was satisfied by the second half + // and would have passed with the split logic deleted. + let ranges = |count: u32| EventQueryFilter { + event_ids: (0..count) .map(|i| EventIdSelector::Range { low: i * 100, high: i * 100 + 50, @@ -988,25 +1094,36 @@ mod expression_budget_tests { .collect(), ..Default::default() }; - let singles_filter = EventQueryFilter { + + // Ten singles cost ten and stay in one node. + let singles_query = build_query(&EventQueryFilter { event_ids: singles(10), ..Default::default() - }; + }) + .expect("builds"); + assert!(!singles_query.contains("")); + + // Ten ranges cost exactly the budget, so they also stay in one node. + let at_budget = build_query(&ranges(10)).expect("builds"); + assert!( + !at_budget.contains(""), + "ten ranges are exactly the budget: {at_budget}" + ); + assert_eq!(at_budget.matches("EventID").count(), 20); - assert!(!build_query(&singles_filter) - .expect("builds") - .contains("")); + // Eleven cost 22 and must split, which is what proves ranges are costed as two. + let over_budget = build_query(&ranges(11)).expect("builds"); assert!( - build_query(&ranges) - .expect("builds") - .contains("") - || build_query(&ranges) - .expect("builds") - .matches("EventID") - .count() - == 20, - "ten ranges must not silently exceed the compound-expression ceiling" + over_budget.starts_with(""), + "eleven ranges exceed the budget and must split: {over_budget}" ); + for node in over_budget.split("").next().unwrap_or_default(); + assert!( + body.matches("EventID").count() <= MAX_EXPRESSIONS_PER_SELECT, + "a node exceeded the budget: {body}" + ); + } } #[test] diff --git a/crates/cmtraceopen-parser/src/eventmap/model.rs b/crates/cmtraceopen-parser/src/eventmap/model.rs index 33d72aafa..c43945fa2 100644 --- a/crates/cmtraceopen-parser/src/eventmap/model.rs +++ b/crates/cmtraceopen-parser/src/eventmap/model.rs @@ -14,6 +14,10 @@ use serde::{Deserialize, Deserializer}; /// The upstream corpus is not perfectly consistent: four maps spell the target `Username` rather /// than `UserName`, so parsing is ASCII case-insensitive. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +// Growable: this build does not know every variant a newer schema will define. +// Marking it now keeps adding one a minor change; after the first release that +// exposes the type, adding the attribute is itself breaking. +#[non_exhaustive] pub enum MapProperty { /// The acting or target account. UserName, diff --git a/crates/cmtraceopen-parser/src/eventmap/path.rs b/crates/cmtraceopen-parser/src/eventmap/path.rs index f4dbcb0b5..f8f3f4f48 100644 --- a/crates/cmtraceopen-parser/src/eventmap/path.rs +++ b/crates/cmtraceopen-parser/src/eventmap/path.rs @@ -24,6 +24,10 @@ use super::node::EventNode; /// A failure to parse a map `Value` expression. #[derive(Debug, Error, PartialEq, Eq)] +// Growable: this build does not know every variant a newer schema will define. +// Marking it now keeps adding one a minor change; after the first release that +// exposes the type, adding the attribute is itself breaking. +#[non_exhaustive] pub enum PathError { /// The expression did not start with `/`. #[error("value path must be absolute, starting with '/': {0}")] diff --git a/crates/cmtraceopen-parser/src/unified_timeline/mod.rs b/crates/cmtraceopen-parser/src/unified_timeline/mod.rs index 70917170e..69bd809ef 100644 --- a/crates/cmtraceopen-parser/src/unified_timeline/mod.rs +++ b/crates/cmtraceopen-parser/src/unified_timeline/mod.rs @@ -116,6 +116,10 @@ pub struct UnplacedItem { /// Why an item has no position on the timeline. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] +// Growable: this build does not know every variant a newer schema will define. +// Marking it now keeps adding one a minor change; after the first release that +// exposes the type, adding the attribute is itself breaking. +#[non_exhaustive] pub enum UnplacedReason { /// The source carried no timestamp, or one the parser could not read. /// diff --git a/crates/cmtraceopen-parser/tests/eventmap_corpus.rs b/crates/cmtraceopen-parser/tests/eventmap_corpus.rs index fd2a1bc15..4189841bd 100644 --- a/crates/cmtraceopen-parser/tests/eventmap_corpus.rs +++ b/crates/cmtraceopen-parser/tests/eventmap_corpus.rs @@ -5,7 +5,9 @@ //! shipped rather than as imagined, so a regression here means the engine has drifted from the //! community format. -use cmtraceopen_parser::eventmap::{apply_map, EventMap, EventNode, MapProperty, MapRegistry}; +use cmtraceopen_parser::eventmap::{ + apply_map, EventMap, EventNode, MapProperty, MapRegistry, ValuePath, +}; const SHELL_CORE_9701: &str = include_str!("fixtures/eventmap/shell-core-9701.json"); const SECURITY_4624: &str = include_str!("fixtures/eventmap/security-4624.json"); @@ -196,14 +198,28 @@ fn a_registry_resolves_each_fixture_by_its_own_identity() { #[test] fn every_fixture_parses_with_no_malformed_paths() { + // Every expression is parsed directly rather than inferred from apply_map. Applying a map + // skips any binding whose %Name% is absent from its template, and a skipped binding never + // reaches the parser, so a malformed expression on one would leave invalid_paths empty and + // this test would pass while proving nothing about that binding. + let mut checked = 0usize; for raw in [SHELL_CORE_9701, SECURITY_4624, NTFS_146] { let map = load(raw); - let mapped = apply_map(&map, &EventNode::new("Event")); - assert!( - mapped.invalid_paths.is_empty(), - "upstream map {} has expressions this engine cannot parse: {:?}", - map.event_id, - mapped.invalid_paths - ); + for entry in &map.maps { + for binding in &entry.values { + assert!( + ValuePath::parse(&binding.value).is_ok(), + "upstream map {} binding {} has an expression this engine cannot parse: {}", + map.event_id, + binding.name, + binding.value + ); + checked += 1; + } + } } + assert!( + checked >= 3, + "the fixtures should contribute several bindings, saw {checked}" + ); } diff --git a/src-tauri/src/event_log/commands.rs b/src-tauri/src/event_log/commands.rs index 4fd8492ba..7d15bd12c 100644 --- a/src-tauri/src/event_log/commands.rs +++ b/src-tauri/src/event_log/commands.rs @@ -233,10 +233,16 @@ pub async fn evtx_load_provider_databases( let providers = state.provider_store.clone(); tokio::task::spawn_blocking( move || -> Result, String> { - providers + // Scanned into a fresh store first, then swapped, so the write lock is held for the + // assignment rather than for the whole directory walk. Any parse in flight would + // otherwise block on its read guard for as long as opening every database takes. Same + // rule the map registry follows above. + let mut loaded = super::provider_db::ProviderStore::default(); + let info = loaded.load_directory(&path)?; + *providers .write() - .map_err(|_| "provider store lock was poisoned".to_string())? - .load_directory(&path) + .map_err(|_| "provider store lock was poisoned".to_string())? = loaded; + Ok(info) }, ) .await diff --git a/src-tauri/src/event_log/maps.rs b/src-tauri/src/event_log/maps.rs index fc9f42092..d4a5f79e7 100644 --- a/src-tauri/src/event_log/maps.rs +++ b/src-tauri/src/event_log/maps.rs @@ -433,6 +433,10 @@ fn property_name(property: &MapProperty) -> String { MapProperty::ExecutableInfo => "ExecutableInfo".to_string(), MapProperty::PayloadData(slot) => format!("PayloadData{slot}"), MapProperty::Other(name) => name.clone(), + // MapProperty is non_exhaustive, so a newer schema can add a target this build predates. + // Naming it after its own debug form keeps the column visible and labelled rather than + // dropping data because the name is unfamiliar. + other => format!("{other:?}"), } } diff --git a/src-tauri/src/event_log/parser.rs b/src-tauri/src/event_log/parser.rs index a918d0a64..67bc9eb51 100644 --- a/src-tauri/src/event_log/parser.rs +++ b/src-tauri/src/event_log/parser.rs @@ -143,8 +143,10 @@ fn parse_single_file( let maps = maps .read() .map_err(|_| "map registry lock was poisoned".to_string())?; - let mut providers = providers - .write() + // A read guard: looking a provider up caches internally, so it needs no exclusive access. + // Taking the write lock here blocked every other reader for the length of the file. + let providers = providers + .read() .map_err(|_| "provider store lock was poisoned".to_string())?; // XML rather than JSON. The JSON projection cannot be re-parsed into an event tree, which is @@ -223,7 +225,7 @@ fn parse_single_file( // A provider database, when one is loaded, turns raw field values into the sentence the // provider intended. Without it the file path can only summarise EventData, which is what // every other cross-platform reader shows and why they are hard to read. - let message = describe_event(&mut providers, &provider, event_id, &insertions) + let message = describe_event(&providers, &provider, event_id, &insertions) .or(payload) .unwrap_or_else(|| build_message(&fields)); @@ -359,7 +361,7 @@ struct EventFields { /// insertions the event did not supply, the metadata and the event disagree, and a sentence with /// `%4` embedded in it is less honest than the field summary it would replace. fn describe_event( - store: &mut ProviderStore, + store: &ProviderStore, provider: &str, event_id: u32, insertions: &[String], @@ -690,24 +692,24 @@ mod description_tests { fn with_no_database_loaded_it_falls_back_to_the_field_summary() { // The common case until an operator loads metadata. Must not fail or blank the message. let data = insertions(&[("HRESULT", "0x80180005")]); - assert!(describe_event(&mut empty_store(), "Nobody-Has-This-Provider", 1, &data).is_none()); + assert!(describe_event(&empty_store(), "Nobody-Has-This-Provider", 1, &data).is_none()); } #[test] fn an_unknown_event_id_falls_back_rather_than_inventing_a_description() { let data = insertions(&[("X", "1")]); - assert!(describe_event(&mut empty_store(), "Still-Not-Loaded", 999_999, &data).is_none()); + assert!(describe_event(&empty_store(), "Still-Not-Loaded", 999_999, &data).is_none()); } #[test] #[ignore = "requires a real provider database via CMTRACEOPEN_PROVIDER_DB"] fn a_loaded_database_renders_a_real_provider_description() { // The whole chain: SQLite on disk, gzip payload, provider metadata, insertion rendering. - let mut store = loaded_store(); + let store = loaded_store(); let data = insertions(&[("HRESULT", "0x80180005")]); let described = describe_event( - &mut store, + &store, "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider", 2, &data, @@ -726,11 +728,11 @@ mod description_tests { #[test] #[ignore = "requires a real provider database via CMTRACEOPEN_PROVIDER_DB"] fn an_event_the_database_does_not_cover_still_falls_back() { - let mut store = loaded_store(); + let store = loaded_store(); // A provider that genuinely is not in a Windows capture. assert!(describe_event( - &mut store, + &store, "Definitely-Not-A-Real-Provider", 1, &insertions(&[("a", "b")]) diff --git a/src-tauri/src/event_log/provider_db.rs b/src-tauri/src/event_log/provider_db.rs index 0d038395d..ceb92e4c2 100644 --- a/src-tauri/src/event_log/provider_db.rs +++ b/src-tauri/src/event_log/provider_db.rs @@ -22,6 +22,7 @@ use std::collections::HashMap; use std::io::Read; use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; use cmtraceopen_parser::provider::ProviderMetadata; use flate2::read::GzDecoder; @@ -198,7 +199,14 @@ pub struct ProviderStore { databases: Vec, /// Lowercased provider name to metadata, including negative results so a provider absent from /// every database is not looked up again for every event that mentions it. - cache: HashMap>, + /// + /// Behind an `Arc` because a lookup happens once per record. A real provider carries every + /// event it defines with its description strings, so returning it by value meant a deep clone + /// per event: up to a hundred thousand of them to render one file. + /// Behind a `Mutex` so a lookup does not need `&mut self`. Without it the parse path had to + /// hold a write guard on the whole store for the length of a file, blocking every other reader + /// just to populate a cache. + cache: Mutex>>>, info: Vec, } @@ -214,19 +222,27 @@ impl ProviderStore { let mut databases = Vec::new(); let mut info = Vec::new(); - let mut failures = Vec::new(); - - let mut paths: Vec = entries - .filter_map(Result::ok) - .map(|entry| entry.path()) - .filter(|path| { - path.is_file() - && path - .extension() - .and_then(|extension| extension.to_str()) - .is_some_and(|extension| extension.eq_ignore_ascii_case("db")) - }) - .collect(); + let mut failures: Vec = Vec::new(); + + let mut paths: Vec = Vec::new(); + for entry in entries { + // An enumeration error after the directory opened is recorded, not skipped, matching + // the map loader. Dropping it lets a partial set look complete. + match entry { + Ok(entry) => paths.push(entry.path()), + Err(error) => failures.push(format!( + "cannot read an entry in {}: {error}", + directory.display() + )), + } + } + paths.retain(|path| { + path.is_file() + && path + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("db")) + }); paths.sort(); for path in paths { @@ -243,11 +259,21 @@ impl ProviderStore { self.databases = databases; self.info = info.clone(); - self.cache.clear(); + if let Ok(mut cache) = self.cache.lock() { + cache.clear(); + } if info.is_empty() && !failures.is_empty() { return Err(failures.join("; ")); } + // Reported even when something loaded. Returning Ok and dropping four reasons on the floor + // leaves an operator looking at partial provider coverage with no explanation for it. + for failure in &failures { + log::warn!( + "event=provider_db_skipped directory=\"{}\" reason=\"{failure}\"", + directory.display() + ); + } Ok(info) } @@ -256,10 +282,12 @@ impl ProviderStore { /// Takes `&mut self` because a lookup populates the cache, including the negative result. That /// is what stops a provider absent from every database being searched again for every event /// that names it. - pub fn provider(&mut self, provider_name: &str) -> Option { + pub fn provider(&self, provider_name: &str) -> Option> { let key = provider_name.to_ascii_lowercase(); - if let Some(cached) = self.cache.get(&key) { - return cached.clone(); + if let Ok(cache) = self.cache.lock() { + if let Some(cached) = cache.get(&key) { + return cached.clone(); + } } let mut found = None; @@ -272,7 +300,10 @@ impl ProviderStore { } } - self.cache.insert(key, found.clone()); + let found = found.map(Arc::new); + if let Ok(mut cache) = self.cache.lock() { + cache.insert(key, found.clone()); + } found } diff --git a/src/workspaces/event-log/EvtxTimeline.tsx b/src/workspaces/event-log/EvtxTimeline.tsx index ca6671f08..3e97cf6e9 100644 --- a/src/workspaces/event-log/EvtxTimeline.tsx +++ b/src/workspaces/event-log/EvtxTimeline.tsx @@ -262,8 +262,17 @@ export function EvtxTimeline() { ref={virtualizer.measureElement} data-index={virtualRow.index} role="button" - tabIndex={-1} + // Focusable and activated by keyboard. It was reachable only by pointer, so a + // keyboard user could not expand or collapse a group at all. + tabIndex={0} + aria-expanded={!row.collapsed} onClick={() => toggleGroup(row.key)} + onKeyDown={(event) => { + if (event.key === "Enter" || event.key === " ") { + event.preventDefault(); + toggleGroup(row.key); + } + }} style={{ // Normal flow, matching the record rows. The wrapper above is already // translated to the first visible row's offset and its children stack inside diff --git a/src/workspaces/event-log/EvtxTimelineRow.tsx b/src/workspaces/event-log/EvtxTimelineRow.tsx index 21678c1db..7b3e60342 100644 --- a/src/workspaces/event-log/EvtxTimelineRow.tsx +++ b/src/workspaces/event-log/EvtxTimelineRow.tsx @@ -129,7 +129,12 @@ export const EvtxTimelineRow = memo( style={ isDescription ? { - flex: 1, + // Absorbs the remaining width only while no width has been set for it. + // Ignoring an override meant the Description column could be resized in the + // chooser and never actually change. + ...(width != null + ? { width: `${width}px`, flexShrink: 0 } + : { flex: 1 }), fontSize: `${fontSize}px`, fontWeight: isSelected ? 600 : 400, color: isSelected From 722e79c86f6af059091d4f760aea619523cb8d96 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 10 Aug 2026 00:32:57 -0400 Subject: [PATCH 50/85] fix(parser): bound the terms that repeat in every node, not just the Event IDs The third instance of one class, and this time the class is closed rather than the case. Levels, providers, the time window and the keyword mask are emitted into every node, so chunking the Event IDs can never bring them under the limit. A filter with thirty levels and no Event IDs skipped both the split and suppression paths entirely and emitted one node well past the 24-expression cliff measured earlier. Production sets EvtQueryTolerateQueryErrors, so that rejection reads as a channel with no events: the same silent-empty-result the suppression fix addressed, reached by a different route. An existing test constructed exactly that shape and asserted only the node count, so it passed while pinning a query the service refuses. The property test could not catch it either, because its largest level list held five entries. Two changes, in the order that matters: - levels and providers are deduplicated, providers case-insensitively as the service matches them. Thirty levels drawn from six distinct values cost thirty expressions to say six things. - when the unsplittable terms still exceed one node, the builder returns QueryBuildError::FilterTooComplex rather than emitting something that will be refused. An error the caller can show beats a filter that appears to work and returns nothing. The property test now runs level lists above the budget and checks suppression nodes as well as selections, so this cannot reappear by a third route. Also from the same review: - PayloadEncoding and QueryBuildError are non_exhaustive, matching the three enums marked in the previous commit - the path engine's attribute branch reads only the first repeated element while text content joins them; the asymmetry is deliberate and was untested, so a change making attributes join too would have passed - queryChannels dropped the coverage gaps the backend reported, and a refresh kept stale gaps while dropping new ones. Both load paths now carry them, and a refresh clears them with the records they describe. - the frontend filter type claimed to mirror the backend's while declaring three of its seven fields. Renamed to say it is a subset, so a missing field reads as absent UI rather than absent backend support. Co-Authored-By: Claude Opus 5 --- .../src/event_payload/mod.rs | 3 + .../cmtraceopen-parser/src/event_query/mod.rs | 163 ++++++++++++++++-- .../cmtraceopen-parser/src/eventmap/path.rs | 15 ++ .../event-log/evtx-coverage.test.ts | 18 ++ src/workspaces/event-log/evtx-store.ts | 11 +- src/workspaces/event-log/types.ts | 13 +- 6 files changed, 205 insertions(+), 18 deletions(-) diff --git a/crates/cmtraceopen-parser/src/event_payload/mod.rs b/crates/cmtraceopen-parser/src/event_payload/mod.rs index f533582bb..c9a84baa8 100644 --- a/crates/cmtraceopen-parser/src/event_payload/mod.rs +++ b/crates/cmtraceopen-parser/src/event_payload/mod.rs @@ -16,6 +16,9 @@ use crate::eventmap::EventNode; /// How the payload bytes were interpreted. +// Growable: UTF-16BE and single-byte code pages are both plausible additions. Marking it now keeps +// adding one a minor change; after the first release that exposes the type it is itself breaking. +#[non_exhaustive] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PayloadEncoding { /// UTF-16 little endian, which is what Windows writes most often. diff --git a/crates/cmtraceopen-parser/src/event_query/mod.rs b/crates/cmtraceopen-parser/src/event_query/mod.rs index 3e03dfc6d..2979289ea 100644 --- a/crates/cmtraceopen-parser/src/event_query/mod.rs +++ b/crates/cmtraceopen-parser/src/event_query/mod.rs @@ -206,10 +206,27 @@ fn quote_literal(value: &str) -> Result { /// Why a filter could not be compiled into a query. #[derive(Debug, Clone, PartialEq, Eq, Error)] +#[non_exhaustive] pub enum QueryBuildError { /// The value contains both `'` and `"`, which an XPath 1.0 string literal cannot express. #[error("value contains both quote characters and cannot be an XPath string literal: {0}")] UnquotableValue(String), + /// The terms that cannot be split across nodes already exceed what one node may carry. + /// + /// Levels, providers, the time window and the keyword mask repeat in every node, so no amount + /// of chunking the Event IDs can bring them under the limit. Refused rather than emitted, + /// because the service rejects an oversized query and the tolerate-errors flag turns that + /// refusal into a channel that reports no events at all. + #[error( + "filter needs {needed} expressions before Event IDs, more than the {limit} one query node \ + may carry; narrow the levels or providers" + )] + FilterTooComplex { + /// Expressions the unsplittable terms require. + needed: usize, + /// Expressions one node may carry. + limit: usize, + }, } /// XML-escapes a complete XPath expression for embedding inside a `` document. @@ -273,6 +290,35 @@ fn time_predicate(window: &TimeWindow) -> Result, QueryBuildEr } /// Expressions contributed by everything other than the Event ID list. +/// Levels with duplicates removed, preserving the order they were given in. +/// +/// A caller can legitimately hand the same level twice. Emitting `Level=2 or Level=2` costs two +/// expressions to say one thing, and the budget is small enough that spending it that way pushes a +/// query over the limit for no benefit. +fn distinct_levels(filter: &EventQueryFilter) -> Vec { + let mut seen = Vec::new(); + for level in &filter.levels { + if !seen.contains(level) { + seen.push(*level); + } + } + seen +} + +/// Provider names with duplicates removed, compared case-insensitively as the service matches. +fn distinct_providers(filter: &EventQueryFilter) -> Vec { + let mut seen: Vec = Vec::new(); + for provider in &filter.providers { + if !seen + .iter() + .any(|existing| existing.eq_ignore_ascii_case(provider)) + { + seen.push(provider.clone()); + } + } + seen +} + fn fixed_expression_cost(filter: &EventQueryFilter) -> usize { let time = filter .time @@ -280,7 +326,9 @@ fn fixed_expression_cost(filter: &EventQueryFilter) -> usize { .and_then(|window| time_predicate(window).ok().flatten()) .map(|predicate| predicate.expressions) .unwrap_or(0); - time + filter.levels.len() + filter.providers.len() + usize::from(filter.keywords.is_some()) + time + distinct_levels(filter).len() + + distinct_providers(filter).len() + + usize::from(filter.keywords.is_some()) } fn system_predicates( @@ -295,9 +343,9 @@ fn system_predicates( } } - if !filter.levels.is_empty() { - let levels: Vec = filter - .levels + let levels = distinct_levels(filter); + if !levels.is_empty() { + let levels: Vec = levels .iter() .map(|level| format!("Level={level}")) .collect(); @@ -316,13 +364,14 @@ fn system_predicates( }); } - if !filter.providers.is_empty() { + let distinct = distinct_providers(filter); + if !distinct.is_empty() { let (operator, joiner) = match filter.provider_mode { SelectorMode::Include => ("=", " or "), SelectorMode::Exclude => ("!=", " and "), }; - let mut providers = Vec::with_capacity(filter.providers.len()); - for provider in &filter.providers { + let mut providers = Vec::with_capacity(distinct.len()); + for provider in &distinct { providers.push(format!("@Name{operator}{}", quote_literal(provider)?)); } predicates.push(format!("Provider[{}]", providers.join(joiner))); @@ -390,6 +439,17 @@ pub fn build_query(filter: &EventQueryFilter) -> Result .map(EventIdSelector::expression_cost) .sum(); + // Refused before anything is built. These terms repeat in every node, so no amount of chunking + // the Event IDs brings them under the limit; emitting anyway produces a query the service + // rejects, and the tolerate-errors flag turns that into a channel reporting no events. An + // error the caller can show beats a filter that appears to work and returns nothing. + if fixed_cost > MAX_EXPRESSIONS_PER_SELECT { + return Err(QueryBuildError::FilterTooComplex { + needed: fixed_cost, + limit: MAX_EXPRESSIONS_PER_SELECT, + }); + } + let over_budget = !filter.event_ids.is_empty() && fixed_cost + event_id_cost > MAX_EXPRESSIONS_PER_SELECT; @@ -1144,8 +1204,9 @@ mod expression_budget_tests { } #[test] - fn a_filter_whose_fixed_terms_fill_the_budget_still_produces_a_query() { - // Pathological but must not loop forever or emit an empty node. + fn repeated_levels_collapse_instead_of_spending_the_budget_twice() { + // The same level given twice would emit "Level=2 or Level=2": two expressions to say one + // thing, out of a budget of twenty. let f = EventQueryFilter { levels: (0..30).map(|n| (n % 6) as u8).collect(), event_ids: singles(3), @@ -1153,8 +1214,62 @@ mod expression_budget_tests { }; let query = build_query(&f).expect("builds"); - assert!(query.contains("").count(), 3, "one id per node"); + assert_eq!( + query.matches("Level=").count(), + 6, + "six distinct levels, however many times each was given: {query}" + ); + } + + #[test] + fn repeated_providers_collapse_case_insensitively() { + // The service matches provider names without regard to case, so two spellings of one name + // are one term. + let f = EventQueryFilter { + providers: vec![ + "Microsoft-Windows-Kernel-General".into(), + "microsoft-windows-kernel-general".into(), + "Another-Provider".into(), + ], + ..Default::default() + }; + + let query = build_query(&f).expect("builds"); + assert_eq!(query.matches("@Name=").count(), 2, "{query}"); + } + + #[test] + fn a_filter_whose_unsplittable_terms_exceed_one_node_is_refused() { + // Levels, providers, time and keywords repeat in every node, so chunking the Event IDs + // cannot bring them under the limit. Emitting anyway produces a query the service rejects, + // and EvtQueryTolerateQueryErrors turns that refusal into a channel reporting no events: + // the filter looks like it worked and returns nothing. An error the caller can show is the + // only honest outcome. + let f = EventQueryFilter { + providers: (0..30).map(|n| format!("Provider-{n}")).collect(), + event_ids: singles(3), + ..Default::default() + }; + + match build_query(&f) { + Err(QueryBuildError::FilterTooComplex { needed, limit }) => { + assert_eq!(needed, 30); + assert_eq!(limit, MAX_EXPRESSIONS_PER_SELECT); + } + other => panic!("expected a refusal, got {other:?}"), + } + } + + #[test] + fn a_filter_exactly_at_the_budget_is_still_built() { + // The boundary is inclusive: twenty is what a node may carry, so it must not be refused. + let f = EventQueryFilter { + providers: (0..MAX_EXPRESSIONS_PER_SELECT) + .map(|n| format!("Provider-{n}")) + .collect(), + ..Default::default() + }; + assert!(build_query(&f).is_ok()); } } @@ -1359,8 +1474,16 @@ mod expression_budget_service_tests { fn no_emitted_node_can_exceed_the_budget() { // The property that matters, checked across shapes rather than for one case: whatever the // filter, every node the builder emits stays inside the budget. - for id_count in [1usize, 5, 19, 20, 21, 40, 100] { - for levels in [vec![], vec![1, 2], vec![1, 2, 3, 4, 5]] { + // Level lists past the budget included, since the gap this test missed the first time was + // a filter whose unsplittable terms alone exceeded a node. + for id_count in [0usize, 1, 5, 19, 20, 21, 40, 100] { + for levels in [ + vec![], + vec![1, 2], + vec![1, 2, 3, 4, 5], + (0..30).map(|n| (n % 6) as u8).collect(), + (0..40u8).collect(), + ] { let filter = EventQueryFilter { time: Some(TimeWindow::Between { from: Some("2020-01-01T00:00:00.000Z".into()), @@ -1373,7 +1496,11 @@ mod expression_budget_service_tests { keywords: Some(0x8020_0000_0000_0000), ..Default::default() }; - let query = build_query(&filter).expect("builds"); + // A refusal is a correct outcome here: it is what the builder does instead of + // emitting something the service would reject. + let Ok(query) = build_query(&filter) else { + continue; + }; for node in query.split("").next().unwrap_or_default(); // Each comparison is one expression; they are joined by `and` or `or`. @@ -1387,6 +1514,14 @@ mod expression_budget_service_tests { "{comparisons} expressions in one node for {id_count} ids, {levels:?} levels" ); } + // Suppressions are nodes too, and are bounded by the same limit. + for node in query.split("").skip(1) { + let body = node.split("").next().unwrap_or_default(); + assert!( + body.matches("EventID").count() <= MAX_EXPRESSIONS_PER_SELECT, + "a suppression exceeded the budget: {body}" + ); + } } } } diff --git a/crates/cmtraceopen-parser/src/eventmap/path.rs b/crates/cmtraceopen-parser/src/eventmap/path.rs index f8f3f4f48..a07ba86cf 100644 --- a/crates/cmtraceopen-parser/src/eventmap/path.rs +++ b/crates/cmtraceopen-parser/src/eventmap/path.rs @@ -398,4 +398,19 @@ mod tests { let path = ValuePath::parse("/Other/EventData").expect("parses"); assert_eq!(path.evaluate(&event()), None); } + + #[test] + fn an_attribute_selector_reads_only_the_first_repeated_element() { + // Deliberately asymmetric with text content, which joins repeats with ", " to match + // EvtxECmd. Nothing pinned the attribute half, so a change making attributes join too + // would have passed the whole suite. + let event = EventNode::new("Event").with_child( + EventNode::new("EventData") + .with_child(EventNode::new("Data").with_attribute("Name", "first")) + .with_child(EventNode::new("Data").with_attribute("Name", "second")), + ); + + let path = ValuePath::parse("/Event/EventData/Data/@Name").expect("parses"); + assert_eq!(path.evaluate(&event).as_deref(), Some("first")); + } } diff --git a/src/workspaces/event-log/evtx-coverage.test.ts b/src/workspaces/event-log/evtx-coverage.test.ts index 4a85efbfb..5405a4f6f 100644 --- a/src/workspaces/event-log/evtx-coverage.test.ts +++ b/src/workspaces/event-log/evtx-coverage.test.ts @@ -41,3 +41,21 @@ describe("summarizeCoverageGaps", () => { expect(summarizeCoverageGaps(["a", "b"])).toBe("2 gaps in this view"); }); }); + +describe("gaps across load paths", () => { + it("a refresh replaces gaps rather than carrying stale ones forward", () => { + // The refresh clears records, so the gaps describing them have to go too. Keeping them would + // report a gap from a set no longer on screen while the new result's own gap went unsaid. + const beforeRefresh = ["Application: 3 records unreadable"]; + const afterClear = mergeCoverageGaps([], ["System: stopped at 100000 events"]); + expect(afterClear).not.toContain(beforeRefresh[0]); + expect(afterClear).toEqual(["System: stopped at 100000 events"]); + }); + + it("an incremental channel load adds to what is already reported", () => { + // Channels load one at a time, so replacing here would leave only the last channel's gaps. + const first = mergeCoverageGaps([], ["Application: 3 records unreadable"]); + const second = mergeCoverageGaps(first, ["Security: 1 record unreadable"]); + expect(second).toHaveLength(2); + }); +}); diff --git a/src/workspaces/event-log/evtx-store.ts b/src/workspaces/event-log/evtx-store.ts index 2b9568246..7603a35f6 100644 --- a/src/workspaces/event-log/evtx-store.ts +++ b/src/workspaces/event-log/evtx-store.ts @@ -9,7 +9,7 @@ import type { EvtxLevel, EvtxParseResult, EvtxTimeWindow, - EventQueryFilter, + EventQueryFilterSubset, } from "./types"; import { EVTX_TIME_WINDOW_MS } from "./types"; @@ -32,7 +32,7 @@ import { * existing controls filter records already in memory; moving them server-side changes what a * reload fetches, which is a behavioural change worth making deliberately rather than implicitly. */ -function buildServerFilter(timeWindow: EvtxTimeWindow): EventQueryFilter { +function buildServerFilter(timeWindow: EvtxTimeWindow): EventQueryFilterSubset { if (timeWindow === "all") return {}; return { time: { kind: "last", milliseconds: EVTX_TIME_WINDOW_MS[timeWindow] } }; } @@ -287,6 +287,9 @@ export const useEvtxStore = create()((set, get) => ({ loadedChannels: newLoaded, isLoading: false, loadError: null, + // Accumulated, not dropped. This path loads channels incrementally, so discarding what the + // backend reported here would show a complete view of a partly unreadable set. + coverageGaps: mergeCoverageGaps(state.coverageGaps, result.errorMessages), selectedRecordId: null, }); } catch (error) { @@ -316,6 +319,9 @@ export const useEvtxStore = create()((set, get) => ({ isLoading: true, loadStartTime: startTime, loadElapsedMs: null, + // Cleared with the records they describe. Keeping them would report gaps from a set that is + // no longer on screen, while the new results' own gaps went unreported. + coverageGaps: [], }); const promises = loaded.map(async (ch) => { @@ -343,6 +349,7 @@ export const useEvtxStore = create()((set, get) => ({ channels: newChannels, loadedChannels: newLoaded, loadElapsedMs: performance.now() - startTime, + coverageGaps: mergeCoverageGaps(s.coverageGaps, result.errorMessages), }); } catch (e) { console.warn(`[evtx] Refresh failed for ${ch}:`, e); diff --git a/src/workspaces/event-log/types.ts b/src/workspaces/event-log/types.ts index d10d411f9..2ccca949f 100644 --- a/src/workspaces/event-log/types.ts +++ b/src/workspaces/event-log/types.ts @@ -80,8 +80,17 @@ export const EVTX_TIME_WINDOW_LABELS: Record = { all: "All time", }; -/** Mirrors `cmtraceopen_parser::event_query::EventQueryFilter`. */ -export interface EventQueryFilter { +/** + * The subset of the backend's query filter this workspace currently sends. + * + * Deliberately not the whole contract. `cmtraceopen_parser::event_query::EventQueryFilter` also + * carries `eventIds`, `eventIdMode`, `providerMode` and `keywords`, and its `TimeWindow` has a + * `between` variant as well as `last`. The Rust struct is `#[serde(default)]`, so omitting them + * deserializes to defaults and this works; what is absent is UI for them, not backend support. + * + * Named as a subset so nobody reads a missing field here as a capability the backend lacks. + */ +export interface EventQueryFilterSubset { time?: { kind: "last"; milliseconds: number }; levels?: number[]; providers?: string[]; From d44afcbc0e4e10146595dcbe9c23c855015f342a Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 10 Aug 2026 01:11:39 -0400 Subject: [PATCH 51/85] test(event-log): drive coverage gaps through the store, not just the merge rule The existing tests called mergeCoverageGaps with hand-picked arrays. That rule can be correct while a call site discards the gaps entirely, which is exactly what queryChannels did, so those tests would have passed through the defect they were written alongside. These drive the store itself with the Tauri bridge mocked. Verified they can fail: removing the merge from queryChannels fails four of the five. Also moved a doc line that had been stacked onto distinct_levels while describing fixed_expression_cost, which had none. Co-Authored-By: Claude Opus 5 --- .../cmtraceopen-parser/src/event_query/mod.rs | 5 +- .../event-log/evtx-store-coverage.test.ts | 93 +++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 src/workspaces/event-log/evtx-store-coverage.test.ts diff --git a/crates/cmtraceopen-parser/src/event_query/mod.rs b/crates/cmtraceopen-parser/src/event_query/mod.rs index 2979289ea..9c07d6df5 100644 --- a/crates/cmtraceopen-parser/src/event_query/mod.rs +++ b/crates/cmtraceopen-parser/src/event_query/mod.rs @@ -289,7 +289,6 @@ fn time_predicate(window: &TimeWindow) -> Result, QueryBuildEr } } -/// Expressions contributed by everything other than the Event ID list. /// Levels with duplicates removed, preserving the order they were given in. /// /// A caller can legitimately hand the same level twice. Emitting `Level=2 or Level=2` costs two @@ -319,6 +318,10 @@ fn distinct_providers(filter: &EventQueryFilter) -> Vec { seen } +/// Expressions contributed by everything other than the Event ID list. +/// +/// These repeat in every node, so they are what decides whether chunking the Event IDs can bring a +/// query under the limit at all. fn fixed_expression_cost(filter: &EventQueryFilter) -> usize { let time = filter .time diff --git a/src/workspaces/event-log/evtx-store-coverage.test.ts b/src/workspaces/event-log/evtx-store-coverage.test.ts new file mode 100644 index 000000000..de48e9db8 --- /dev/null +++ b/src/workspaces/event-log/evtx-store-coverage.test.ts @@ -0,0 +1,93 @@ +/** + * Coverage-gap handling through the store's real load paths. + * + * `evtx-coverage.test.ts` covers the merge rule in isolation. That is not enough: the rule can be + * right while a call site drops the gaps entirely, which is exactly what `queryChannels` did. These + * drive the store itself so a regression in a load path fails here. + * + * The Tauri bridge is mocked because the store imports it at module scope. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const invoke = vi.hoisted(() => vi.fn()); +vi.mock("@tauri-apps/api/core", () => ({ invoke })); +vi.mock("@tauri-apps/api/event", () => ({ listen: vi.fn().mockResolvedValue(() => {}) })); + +const { useEvtxStore } = await import("./evtx-store"); + +function result(channel: string, gaps: string[]) { + return { + records: [], + channels: [{ name: channel, eventCount: 0, sourceType: "live" }], + totalRecords: 0, + parseErrors: gaps.length, + errorMessages: gaps, + }; +} + +describe("coverage gaps through the store", () => { + beforeEach(() => { + invoke.mockReset(); + useEvtxStore.setState({ + records: [], + channels: [], + coverageGaps: [], + loadedChannels: new Set(), + selectedChannels: new Set(), + }); + }); + + it("keeps the gaps a channel query reported", async () => { + // queryChannels previously discarded these, so a partly unreadable channel looked complete. + invoke.mockResolvedValueOnce(result("Application", ["Application: 3 records unreadable"])); + + await useEvtxStore.getState().queryChannels(["Application"]); + + expect(useEvtxStore.getState().coverageGaps).toEqual([ + "Application: 3 records unreadable", + ]); + }); + + it("accumulates gaps as channels load one at a time", async () => { + invoke + .mockResolvedValueOnce(result("Application", ["Application: 3 records unreadable"])) + .mockResolvedValueOnce(result("System", ["System: stopped at 100000 events"])); + + await useEvtxStore.getState().queryChannels(["Application"]); + await useEvtxStore.getState().queryChannels(["System"]); + + expect(useEvtxStore.getState().coverageGaps).toHaveLength(2); + }); + + it("does not repeat a gap when the same channel is queried again", async () => { + invoke + .mockResolvedValueOnce(result("Application", ["Application: 3 records unreadable"])) + .mockResolvedValueOnce(result("Application", ["Application: 3 records unreadable"])); + + await useEvtxStore.getState().queryChannels(["Application"]); + await useEvtxStore.getState().queryChannels(["Application"]); + + expect(useEvtxStore.getState().coverageGaps).toHaveLength(1); + }); + + it("reports nothing when a channel loads cleanly", async () => { + invoke.mockResolvedValueOnce(result("Application", [])); + + await useEvtxStore.getState().queryChannels(["Application"]); + + expect(useEvtxStore.getState().coverageGaps).toEqual([]); + }); + + it("a refresh drops gaps from the records it replaced", async () => { + // The refresh clears the records, so gaps describing them must go too, or the banner reports a + // gap in a set that is no longer on screen. + invoke.mockResolvedValueOnce(result("Application", ["Application: stale gap"])); + await useEvtxStore.getState().queryChannels(["Application"]); + expect(useEvtxStore.getState().coverageGaps).toHaveLength(1); + + invoke.mockResolvedValueOnce(result("Application", ["Application: fresh gap"])); + await useEvtxStore.getState().refreshLoadedChannels(); + + expect(useEvtxStore.getState().coverageGaps).toEqual(["Application: fresh gap"]); + }); +}); From 21a0c058426af5374e9bfef27f4dda5328284dc4 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 10 Aug 2026 01:54:50 -0400 Subject: [PATCH 52/85] fix(event-log): label positional fields by slot, and use the one filter ordering Two review findings. A positional field's label is how an operator matches it against the provider's template, and the counter was incremented after the emptiness check. For second the surviving field was labelled Data1 while the template addresses it as %2. The rendered message was already correct, since it reads the insertion list; only the label lied, and the doc comment above it claimed the opposite. The filter bar re-implemented the saved-filter ordering and dropped the lastUsed tiebreak, so marking a filter used changed the stored order and never changed what was on screen. It calls orderFilters now, which is the single ordering the store already exposes. Co-Authored-By: Claude Opus 5 --- src-tauri/src/event_log/parser.rs | 35 ++++++++++++++++------ src/workspaces/event-log/EvtxFilterBar.tsx | 10 +++---- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src-tauri/src/event_log/parser.rs b/src-tauri/src/event_log/parser.rs index 67bc9eb51..8302fb5a8 100644 --- a/src-tauri/src/event_log/parser.rs +++ b/src-tauri/src/event_log/parser.rs @@ -310,18 +310,24 @@ fn extract_event_data(root: &cmtraceopen_parser::eventmap::EventNode) -> EventFi // so skipping one here shifts every later %N and renders the description with the wrong // values substituted into it, which reads as fact. insertions.push(value.clone()); + + // Counted before the emptiness check, for the same reason. The label is what an operator + // uses to match a field against the provider's template, so skipping the count for a blank + // slot would label the second field Data1 while the template calls it %2. + let position = if child.attribute("Name").is_none() && child.name == "Data" { + *unnamed += 1; + Some(*unnamed) + } else { + None + }; + if value.is_empty() { return; } - let name = match child.attribute("Name") { - Some(name) => name.to_string(), - // A positional field. Numbered from one so it lines up with the `%1` style insertion - // numbering that message templates use. - None if child.name == "Data" => { - *unnamed += 1; - format!("Data{unnamed}") - } - None => child.name.clone(), + let name = match (child.attribute("Name"), position) { + (Some(name), _) => name.to_string(), + (None, Some(position)) => format!("Data{position}"), + (None, None) => child.name.clone(), }; fields.push(EvtxField { name, value }); }; @@ -538,6 +544,17 @@ mod tests { assert_eq!(insertions_of(xml), vec!["", "second"]); } + #[test] + fn a_positional_label_matches_the_slot_the_template_addresses() { + // The label is how an operator matches a field against the provider's template. Skipping + // the count for a blank slot labelled the survivor Data1 while the template calls it %2. + let xml = "second"; + let fields = fields_of(xml); + assert_eq!(fields.len(), 1); + assert_eq!(fields[0].name, "Data2"); + assert_eq!(fields[0].value, "second"); + } + #[test] fn insertions_cover_user_data_too() { let xml = r#" diff --git a/src/workspaces/event-log/EvtxFilterBar.tsx b/src/workspaces/event-log/EvtxFilterBar.tsx index cf71eda63..a2b3a6455 100644 --- a/src/workspaces/event-log/EvtxFilterBar.tsx +++ b/src/workspaces/event-log/EvtxFilterBar.tsx @@ -4,7 +4,7 @@ import { invoke } from "@tauri-apps/api/core"; import { save } from "@tauri-apps/plugin-dialog"; import { selectVisibleRecords, EVTX_GROUP_LABELS, type EvtxGroupField } from "./evtx-filter"; import { useSavedFilterStore } from "./evtx-filter-store"; -import { sanitizeCriteria } from "./evtx-saved-filters"; +import { orderFilters, sanitizeCriteria } from "./evtx-saved-filters"; import { availableColumns, discoverMappedProperties, @@ -92,10 +92,10 @@ export function EvtxFilterBar() { const savedFilters = useSavedFilterStore((s) => s.savedFilters); const saveFilter = useSavedFilterStore((s) => s.save); const markFilterUsed = useSavedFilterStore((s) => s.markUsed); - const orderedFilters = useMemo( - () => [...savedFilters].sort((a, b) => (a.favorite === b.favorite ? a.name.localeCompare(b.name) : a.favorite ? -1 : 1)), - [savedFilters] - ); + // The one ordering, not a second one. This re-implemented it and dropped the lastUsed + // tiebreak, so marking a filter used changed the stored order and never changed what the + // operator saw. + const orderedFilters = useMemo(() => orderFilters(savedFilters), [savedFilters]); const columnConfig = useEvtxStore((s) => s.columnConfig); const toggleColumnVisible = useEvtxStore((s) => s.toggleColumnVisible); From 9dae0fc6a281cb8f80ae71a9802ea376138d69ea Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 10 Aug 2026 02:42:03 -0400 Subject: [PATCH 53/85] fix(event-log): send event origins over the wire with the keys the UI reads serde's rename_all renames an enum's variants, not the fields inside a struct variant; that needs rename_all_fields. So TimelineOrigin::Event serialized event_id and record_id in snake_case while the timeline view reads origin.eventId and origin.recordId, and every event row on the unified timeline rendered undefined. A test now asserts the exact wire keys, since nothing in Rust or TypeScript could see the mismatch on its own. Also: - fixtures used RING0IVY24-01, a real machine on the bench. The repository rule is that fixtures carry no real device names, and it was in four files. Replaced with TESTHOST-01. - reset() left coverageGaps and timeZoneMode behind. Gaps describe records that are gone, so surviving a reset reports a hole in a set no longer on screen, and a zone left from a previous session silently reinterprets the next one's timestamps. - the "orders favorites first" test favourited the name that already sorted first, and both saves stamp lastUsed from the same clock tick, so the assertion held whether or not toggleFavorite did anything. It now favourites the entry that loses on every other rule, and asserts the order before and after. Co-Authored-By: Claude Opus 5 --- .../src/unified_timeline/mod.rs | 42 ++++++++++++++++++- src-tauri/src/event_log/event_node.rs | 4 +- src-tauri/src/event_log/export.rs | 2 +- src-tauri/src/event_log/parser.rs | 4 +- src-tauri/src/event_log/timeline.rs | 2 +- .../event-log/evtx-filter-store.test.ts | 12 ++++-- src/workspaces/event-log/evtx-store.ts | 5 +++ 7 files changed, 61 insertions(+), 10 deletions(-) diff --git a/crates/cmtraceopen-parser/src/unified_timeline/mod.rs b/crates/cmtraceopen-parser/src/unified_timeline/mod.rs index 69bd809ef..06f6caf27 100644 --- a/crates/cmtraceopen-parser/src/unified_timeline/mod.rs +++ b/crates/cmtraceopen-parser/src/unified_timeline/mod.rs @@ -62,7 +62,14 @@ impl TimelineSeverity { /// Where a timeline item came from. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "camelCase")] +// rename_all covers the variant names; rename_all_fields is what camel-cases the fields inside a +// struct variant. Without the second one, event_id went over the wire as "event_id" while the +// timeline view reads origin.eventId, so every event row rendered undefined. +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase" +)] pub enum TimelineOrigin { /// A line from a parsed text log. Log { @@ -406,4 +413,37 @@ mod tests { other => panic!("expected a log origin, got {other:?}"), } } + + #[test] + fn an_event_origin_serializes_with_the_keys_the_frontend_reads() { + // rename_all on an enum renames the variants, not the fields inside a struct variant, so + // this needs rename_all_fields. Without it event_id went over the wire as "event_id" while + // the timeline view reads origin.eventId, and every event row showed undefined. + let origin = TimelineOrigin::Event { + channel: "Application".into(), + provider: "ESENT".into(), + event_id: 326, + record_id: 42, + }; + let json = serde_json::to_value(&origin).expect("serializes"); + + assert_eq!(json["kind"], "event"); + assert_eq!(json["eventId"], 326); + assert_eq!(json["recordId"], 42); + assert!(json.get("event_id").is_none(), "{json}"); + assert!(json.get("record_id").is_none(), "{json}"); + } + + #[test] + fn a_log_origin_keeps_its_wire_keys() { + let origin = TimelineOrigin::Log { + file: "cmt.log".into(), + component: Some("Agent".into()), + line: 7, + }; + let json = serde_json::to_value(&origin).expect("serializes"); + assert_eq!(json["kind"], "log"); + assert_eq!(json["file"], "cmt.log"); + assert_eq!(json["line"], 7); + } } diff --git a/src-tauri/src/event_log/event_node.rs b/src-tauri/src/event_log/event_node.rs index 42abeaffd..0f50103fc 100644 --- a/src-tauri/src/event_log/event_node.rs +++ b/src-tauri/src/event_log/event_node.rs @@ -214,7 +214,7 @@ mod tests { Security - RING0IVY24-01 + TESTHOST-01 adam @@ -262,7 +262,7 @@ mod tests { fn element_text_resolves() { assert_eq!( resolve(RENDERED, "/Event/System/Computer").as_deref(), - Some("RING0IVY24-01") + Some("TESTHOST-01") ); } diff --git a/src-tauri/src/event_log/export.rs b/src-tauri/src/event_log/export.rs index 612de8844..5ffba6dd8 100644 --- a/src-tauri/src/event_log/export.rs +++ b/src-tauri/src/event_log/export.rs @@ -173,7 +173,7 @@ mod tests { channel: "Application".into(), event_id: 326, level: EvtxLevel::Error, - computer: "RING0IVY24-01".into(), + computer: "TESTHOST-01".into(), message: message.into(), event_data: Vec::new(), raw_xml: "".into(), diff --git a/src-tauri/src/event_log/parser.rs b/src-tauri/src/event_log/parser.rs index 8302fb5a8..e80e14266 100644 --- a/src-tauri/src/event_log/parser.rs +++ b/src-tauri/src/event_log/parser.rs @@ -634,7 +634,7 @@ mod tests { 2 System - RING0IVY24-01 + TESTHOST-01 "#, )); @@ -646,7 +646,7 @@ mod tests { assert_eq!(system.event_id, Some(12)); assert_eq!(system.level, Some(2)); assert_eq!(system.channel.as_deref(), Some("System")); - assert_eq!(system.computer.as_deref(), Some("RING0IVY24-01")); + assert_eq!(system.computer.as_deref(), Some("TESTHOST-01")); assert_eq!(system.process_id, Some(4)); assert_eq!( parse_timestamp_to_epoch_ms(system.time_created.as_deref().unwrap_or_default()), diff --git a/src-tauri/src/event_log/timeline.rs b/src-tauri/src/event_log/timeline.rs index a35450349..55a162d7b 100644 --- a/src-tauri/src/event_log/timeline.rs +++ b/src-tauri/src/event_log/timeline.rs @@ -96,7 +96,7 @@ mod tests { .to_string(), event_id: 76, level, - computer: "RING0IVY24-01".to_string(), + computer: "TESTHOST-01".to_string(), message: message.to_string(), event_data: Vec::new(), raw_xml: String::new(), diff --git a/src/workspaces/event-log/evtx-filter-store.test.ts b/src/workspaces/event-log/evtx-filter-store.test.ts index 33e9d1ff4..3e3bba7eb 100644 --- a/src/workspaces/event-log/evtx-filter-store.test.ts +++ b/src/workspaces/event-log/evtx-filter-store.test.ts @@ -43,11 +43,17 @@ describe("useSavedFilterStore", () => { }); it("orders favorites first", () => { + // Favourite the one that loses on every other rule. Favouriting "Alpha" proved nothing: both + // saves stamp lastUsed from the same clock tick, so the ordering fell through to the name + // comparison and "Alpha" came first whether or not toggleFavorite did anything at all. const store = useSavedFilterStore.getState(); - store.save("Zulu", criteria()); - const alpha = useSavedFilterStore.getState().save("Alpha", criteria()); - useSavedFilterStore.getState().toggleFavorite(alpha.id); + const zulu = store.save("Zulu", criteria()); + useSavedFilterStore.getState().save("Alpha", criteria()); + expect(useSavedFilterStore.getState().ordered()[0].name).toBe("Alpha"); + + useSavedFilterStore.getState().toggleFavorite(zulu.id); + expect(useSavedFilterStore.getState().ordered()[0].name).toBe("Zulu"); }); it("drops persisted entries that no longer validate rather than repairing them", () => { diff --git a/src/workspaces/event-log/evtx-store.ts b/src/workspaces/event-log/evtx-store.ts index 7603a35f6..3bbd96d3c 100644 --- a/src/workspaces/event-log/evtx-store.ts +++ b/src/workspaces/event-log/evtx-store.ts @@ -440,6 +440,11 @@ export const useEvtxStore = create()((set, get) => ({ filterEventIds: "", filterSearch: "", timeWindow: "24h", + // Reset with everything else. Gaps describe records that are gone, so surviving a reset + // would report a hole in a set no longer on screen, and a zone left over from a previous + // session would silently reinterpret the next one's timestamps. + coverageGaps: [], + timeZoneMode: "local", columnConfig: defaultColumnConfig(), groupBy: [], collapsedGroups: new Set(), From 79de6e2fcf20e49943d7dedbf89487346d21ae19 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 10 Aug 2026 03:24:59 -0400 Subject: [PATCH 54/85] test(event-log): give the timeline fixture distinguishable ids, and document severities The fixture used 76 for both event_record_id and event_id, so a test asserting the origin mapping passed whether or not the two were wired to the right fields. They are now 1234 and 76, and the assertion checks each independently. Also from the same review: TimelineOrigin is non_exhaustive, matching the other growable enums in the crate; TimelineSeverity's Info, Warning and Error variants are documented like the two that already were; and the originDetail test asserts all four details its name promises rather than two. Co-Authored-By: Claude Opus 5 --- crates/cmtraceopen-parser/src/unified_timeline/mod.rs | 6 ++++++ src-tauri/src/event_log/timeline.rs | 6 ++++-- src/workspaces/event-log/unified-timeline.test.ts | 4 ++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/crates/cmtraceopen-parser/src/unified_timeline/mod.rs b/crates/cmtraceopen-parser/src/unified_timeline/mod.rs index 06f6caf27..286e3836f 100644 --- a/crates/cmtraceopen-parser/src/unified_timeline/mod.rs +++ b/crates/cmtraceopen-parser/src/unified_timeline/mod.rs @@ -23,9 +23,12 @@ use crate::models::log_entry::{LogEntry, Severity}; pub enum TimelineSeverity { /// Diagnostic detail, from a verbose event. Text logs have no equivalent. Verbose, + /// Ordinary progress. The default, since a line carrying no severity is not a problem. #[default] Info, + /// Something the source flagged as worth noticing but did not treat as a failure. Warning, + /// A failure the source reported. The highest a text log reaches. Error, /// Only events carry this; text logs top out at error. Critical, @@ -62,6 +65,9 @@ impl TimelineSeverity { /// Where a timeline item came from. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +// Growable: a third kind of source, such as a registry export or an ETW trace read directly, is a +// plausible addition. Marking it now keeps that a minor change. +#[non_exhaustive] // rename_all covers the variant names; rename_all_fields is what camel-cases the fields inside a // struct variant. Without the second one, event_id went over the wire as "event_id" while the // timeline view reads origin.eventId, so every event row rendered undefined. diff --git a/src-tauri/src/event_log/timeline.rs b/src-tauri/src/event_log/timeline.rs index 55a162d7b..36d2c4f75 100644 --- a/src-tauri/src/event_log/timeline.rs +++ b/src-tauri/src/event_log/timeline.rs @@ -87,7 +87,8 @@ mod tests { fn record(timestamp_epoch: i64, message: &str, level: EvtxLevel) -> EvtxRecord { EvtxRecord { id: 0, - event_record_id: 76, + // Distinct from event_id below, so a test cannot pass while reading the wrong one. + event_record_id: 1234, timestamp: String::new(), timestamp_epoch, provider: "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider" @@ -168,8 +169,9 @@ mod tests { .. } => { assert!(channel.contains("DeviceManagement")); + // Distinct values, so mapping event_id to record_id or the reverse fails here. assert_eq!(*event_id, 76); - assert_eq!(*record_id, 76); + assert_eq!(*record_id, 1234); } other => panic!("expected an event origin, got {other:?}"), } diff --git a/src/workspaces/event-log/unified-timeline.test.ts b/src/workspaces/event-log/unified-timeline.test.ts index 618de93ca..1ff7629c4 100644 --- a/src/workspaces/event-log/unified-timeline.test.ts +++ b/src/workspaces/event-log/unified-timeline.test.ts @@ -57,7 +57,11 @@ describe("originDetail", () => { }); it("gives channel, provider, event and record for an event", () => { + // All four, as the name promises. Asserting only two let a change that dropped the channel or + // the provider from the detail line pass. const detail = originDetail(eventOrigin); + expect(detail).toContain(eventOrigin.channel); + expect(detail).toContain(eventOrigin.provider); expect(detail).toContain("event 76"); expect(detail).toContain("record 1234"); }); From cb2b13ca0a3ebe7fc62dbb976702eca8b2ad43b6 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 10 Aug 2026 09:17:08 -0400 Subject: [PATCH 55/85] fix(event-log): apply the time window on refresh, and stop emitting over-budget nodes Both blocking findings from the Hermes charter review on #541. The time-window control was a no-op for live channels. It is a server-side predicate, so only a refetch applies it, and the toolbar calls refreshLoadedChannels after changing it. That refresh sent channels and maxEvents but no filter, so selecting 1h refetched the channel unbounded and filled the view with events outside the window the toolbar still showed as selected. Worse than an ignored setting: the view claimed a constraint it did not have. All three query paths now send the filter, and a test asserts the window reaches the service on both the initial query and the refresh. The chunker's floor let a single selector exceed the remaining budget. A selector cannot be split -- a range is two comparisons and stays together -- so with nineteen providers the budget was one and a range costing two was emitted anyway, producing a node of twenty-one expressions against a budget of twenty. Measured: 19 providers plus a range emits 21, and 20 plus a range emits 22. Neither crosses the 24 the service was measured to refuse, so nothing was being rejected today. The budget is not the limit, it is the headroom held under the limit, and that headroom is what absorbed two earlier miscounts in this same module. Spending it silently is the defect. The refusal now accounts for the largest single selector alongside the fixed terms, and the chunker has no fallback that emits an oversized node. The property test covers ranges as well as single ids, which is exactly why this went unseen: it only ever built singles. Both fixes are mutation-tested. Restoring the old chunking floor fails two tests; removing the filter from the refresh fails one. Co-Authored-By: Claude Opus 5 --- .../cmtraceopen-parser/src/event_query/mod.rs | 154 +++++++++++++----- .../event-log/evtx-store-coverage.test.ts | 48 ++++++ src/workspaces/event-log/evtx-store.ts | 5 + 3 files changed, 167 insertions(+), 40 deletions(-) diff --git a/crates/cmtraceopen-parser/src/event_query/mod.rs b/crates/cmtraceopen-parser/src/event_query/mod.rs index 9c07d6df5..1d7f14139 100644 --- a/crates/cmtraceopen-parser/src/event_query/mod.rs +++ b/crates/cmtraceopen-parser/src/event_query/mod.rs @@ -322,6 +322,19 @@ fn distinct_providers(filter: &EventQueryFilter) -> Vec { /// /// These repeat in every node, so they are what decides whether chunking the Event IDs can bring a /// query under the limit at all. +/// The most expressions any single Event ID selector costs. +/// +/// A selector cannot be split: a range is two comparisons and stays together. So if the budget +/// left after the fixed terms is smaller than this, no chunking can produce a conforming node. +fn largest_selector_cost(filter: &EventQueryFilter) -> usize { + filter + .event_ids + .iter() + .map(EventIdSelector::expression_cost) + .max() + .unwrap_or(0) +} + fn fixed_expression_cost(filter: &EventQueryFilter) -> usize { let time = filter .time @@ -399,13 +412,15 @@ fn select_body( } /// Splits Event ID selectors so each group fits the expression budget alongside the fixed terms. +/// +/// Every returned chunk costs at most `MAX_EXPRESSIONS_PER_SELECT - fixed_cost`. The caller must +/// have already established that no single selector exceeds that, which +/// [`largest_selector_cost`] is for; there is no fallback that emits an oversized node. fn chunk_by_expression_budget( selectors: &[EventIdSelector], fixed_cost: usize, ) -> Vec> { - // At least one selector per node even when the fixed terms alone fill the budget, so a - // pathological filter still produces a query rather than an empty or infinite split. - let budget = MAX_EXPRESSIONS_PER_SELECT.saturating_sub(fixed_cost).max(1); + let budget = MAX_EXPRESSIONS_PER_SELECT.saturating_sub(fixed_cost); let mut chunks: Vec> = Vec::new(); let mut current: Vec = Vec::new(); let mut spent = 0usize; @@ -446,9 +461,14 @@ pub fn build_query(filter: &EventQueryFilter) -> Result // the Event IDs brings them under the limit; emitting anyway produces a query the service // rejects, and the tolerate-errors flag turns that into a channel reporting no events. An // error the caller can show beats a filter that appears to work and returns nothing. - if fixed_cost > MAX_EXPRESSIONS_PER_SELECT { + // The unsplittable terms plus the largest single selector, since a selector cannot be divided: + // a range is two comparisons and stays together. Checking only the fixed terms left a gap where + // nineteen providers plus one range emitted a node of twenty-one expressions, because the + // chunker's old floor handed out a budget of one to a selector that costs two. + let indivisible = fixed_cost + largest_selector_cost(filter); + if indivisible > MAX_EXPRESSIONS_PER_SELECT { return Err(QueryBuildError::FilterTooComplex { - needed: fixed_cost, + needed: indivisible, limit: MAX_EXPRESSIONS_PER_SELECT, }); } @@ -1256,13 +1276,54 @@ mod expression_budget_tests { match build_query(&f) { Err(QueryBuildError::FilterTooComplex { needed, limit }) => { - assert_eq!(needed, 30); + // Thirty providers plus the largest single selector, which is one here. + assert_eq!(needed, 31); + assert_eq!(limit, MAX_EXPRESSIONS_PER_SELECT); + } + other => panic!("expected a refusal, got {other:?}"), + } + } + + #[test] + fn a_range_that_cannot_fit_beside_the_fixed_terms_is_refused() { + // A selector cannot be split: a range is two comparisons and stays together. Nineteen + // providers leave a budget of one, and the chunker used to hand that one out anyway, + // emitting a node of twenty-one expressions. The budget exists to hold headroom under the + // measured cliff, so spending it silently is what the refusal now prevents. + let filter = EventQueryFilter { + providers: (0..19).map(|n| format!("Provider-{n}")).collect(), + event_ids: vec![EventIdSelector::Range { + low: 1000, + high: 1050, + }], + ..Default::default() + }; + + match build_query(&filter) { + Err(QueryBuildError::FilterTooComplex { needed, limit }) => { + assert_eq!(needed, 21, "19 fixed plus a range costing 2"); assert_eq!(limit, MAX_EXPRESSIONS_PER_SELECT); } other => panic!("expected a refusal, got {other:?}"), } } + #[test] + fn a_range_that_just_fits_beside_the_fixed_terms_is_built() { + // Eighteen leaves a budget of two, which is exactly a range. The boundary must build. + let filter = EventQueryFilter { + providers: (0..18).map(|n| format!("Provider-{n}")).collect(), + event_ids: vec![EventIdSelector::Range { + low: 1000, + high: 1050, + }], + ..Default::default() + }; + let query = build_query(&filter).expect("builds"); + let expressions = query.matches("EventID").count() + query.matches("@Name").count(); + assert_eq!(expressions, MAX_EXPRESSIONS_PER_SELECT); + } + #[test] fn a_filter_exactly_at_the_budget_is_still_built() { // The boundary is inclusive: twenty is what a node may carry, so it must not be refused. @@ -1487,43 +1548,56 @@ mod expression_budget_service_tests { (0..30).map(|n| (n % 6) as u8).collect(), (0..40u8).collect(), ] { - let filter = EventQueryFilter { - time: Some(TimeWindow::Between { - from: Some("2020-01-01T00:00:00.000Z".into()), - to: Some("2030-01-01T00:00:00.000Z".into()), - }), - levels: levels.clone(), - event_ids: (0..id_count as u32) - .map(|id| EventIdSelector::Single { id: 1000 + id }) - .collect(), - keywords: Some(0x8020_0000_0000_0000), - ..Default::default() - }; - // A refusal is a correct outcome here: it is what the builder does instead of - // emitting something the service would reject. - let Ok(query) = build_query(&filter) else { - continue; - }; - for node in query.split("").next().unwrap_or_default(); - // Each comparison is one expression; they are joined by `and` or `or`. - let comparisons = body.matches("EventID").count() - + body.matches("Level=").count() - + body.matches("@SystemTime").count() - + body.matches("timediff").count() - + body.matches("band").count(); - assert!( + // Ranges as well as singles. Covering only singles is precisely why a range being + // handed a budget of one went unnoticed. + for use_ranges in [false, true] { + let filter = EventQueryFilter { + time: Some(TimeWindow::Between { + from: Some("2020-01-01T00:00:00.000Z".into()), + to: Some("2030-01-01T00:00:00.000Z".into()), + }), + levels: levels.clone(), + event_ids: (0..id_count as u32) + .map(|id| { + if use_ranges { + EventIdSelector::Range { + low: 1000 + id * 10, + high: 1000 + id * 10 + 5, + } + } else { + EventIdSelector::Single { id: 1000 + id } + } + }) + .collect(), + keywords: Some(0x8020_0000_0000_0000), + ..Default::default() + }; + // A refusal is a correct outcome here: it is what the builder does instead of + // emitting something the service would reject. + let Ok(query) = build_query(&filter) else { + continue; + }; + for node in query.split("").next().unwrap_or_default(); + // Each comparison is one expression; they are joined by `and` or `or`. + let comparisons = body.matches("EventID").count() + + body.matches("Level=").count() + + body.matches("@SystemTime").count() + + body.matches("timediff").count() + + body.matches("band").count(); + assert!( comparisons <= MAX_EXPRESSIONS_PER_SELECT, "{comparisons} expressions in one node for {id_count} ids, {levels:?} levels" ); - } - // Suppressions are nodes too, and are bounded by the same limit. - for node in query.split("").skip(1) { - let body = node.split("").next().unwrap_or_default(); - assert!( - body.matches("EventID").count() <= MAX_EXPRESSIONS_PER_SELECT, - "a suppression exceeded the budget: {body}" - ); + } + // Suppressions are nodes too, and are bounded by the same limit. + for node in query.split("").skip(1) { + let body = node.split("").next().unwrap_or_default(); + assert!( + body.matches("EventID").count() <= MAX_EXPRESSIONS_PER_SELECT, + "a suppression exceeded the budget: {body}" + ); + } } } } diff --git a/src/workspaces/event-log/evtx-store-coverage.test.ts b/src/workspaces/event-log/evtx-store-coverage.test.ts index de48e9db8..e9198ec0a 100644 --- a/src/workspaces/event-log/evtx-store-coverage.test.ts +++ b/src/workspaces/event-log/evtx-store-coverage.test.ts @@ -91,3 +91,51 @@ describe("coverage gaps through the store", () => { expect(useEvtxStore.getState().coverageGaps).toEqual(["Application: fresh gap"]); }); }); + +describe("the time window reaches the service", () => { + beforeEach(() => { + invoke.mockReset(); + useEvtxStore.setState({ + records: [], + channels: [], + coverageGaps: [], + loadedChannels: new Set(), + selectedChannels: new Set(), + timeWindow: "1h", + }); + }); + + const filterOf = (call: number) => + (invoke.mock.calls[call]?.[1] as { filter?: { time?: { milliseconds: number } } })?.filter; + + it("sends the selected window when a channel is queried", () => { + invoke.mockResolvedValueOnce(result("Application", [])); + return useEvtxStore + .getState() + .queryChannels(["Application"]) + .then(() => { + expect(filterOf(0)?.time?.milliseconds).toBe(60 * 60 * 1000); + }); + }); + + it("sends it on refresh too", async () => { + // The window is a server-side predicate and a refresh is the only thing that applies it. + // Omitting it made the control a no-op: selecting 1h triggered a refresh that then fetched the + // channel unbounded, so the view filled with events outside the window still shown as selected. + invoke.mockResolvedValueOnce(result("Application", [])); + await useEvtxStore.getState().queryChannels(["Application"]); + + invoke.mockResolvedValueOnce(result("Application", [])); + await useEvtxStore.getState().refreshLoadedChannels(); + + expect(filterOf(1)?.time?.milliseconds).toBe(60 * 60 * 1000); + }); + + it("sends no time predicate when the window is all time", async () => { + useEvtxStore.setState({ timeWindow: "all" }); + invoke.mockResolvedValueOnce(result("Application", [])); + await useEvtxStore.getState().queryChannels(["Application"]); + + expect(filterOf(0)?.time).toBeUndefined(); + }); +}); diff --git a/src/workspaces/event-log/evtx-store.ts b/src/workspaces/event-log/evtx-store.ts index 3bbd96d3c..291d8200e 100644 --- a/src/workspaces/event-log/evtx-store.ts +++ b/src/workspaces/event-log/evtx-store.ts @@ -329,6 +329,11 @@ export const useEvtxStore = create()((set, get) => ({ const result = await invoke("evtx_query_channels", { channels: [ch], maxEvents: null, + // The window is a server-side predicate and a refetch is the only thing that applies it. + // Omitting it here made the time-window control a no-op: selecting 1h triggered this + // refresh, which then fetched the channel unbounded and replaced the view with events + // outside the window the toolbar was still showing as selected. + filter: buildServerFilter(get().timeWindow), }); const s = get(); From 48dce30e45a51a3f766eefaac30c6f9c47a607f6 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 10 Aug 2026 10:06:31 -0400 Subject: [PATCH 56/85] fix(event-log): close four saved-filter findings I had misread as already fixed CodeRabbit lists a finding it has raised before under "Duplicate comments". I had been reading that as "already handled" and skipping them. It means the finding is unchanged from a previous round, and these four were never addressed at all. That misreading is why the review kept coming back. - a whitespace-only filter name saved fine and then vanished on restart, because sanitizeSavedFilter drops it on rehydration. It looked like the app losing the operator's work. save() now refuses it and returns null. - lastUsed accepted any number. JSON admits 1e309, which parses to Infinity and reaches the ordering comparator as a non-finite operand. Finite only. - parseFilterExport ignored the schema version that buildFilterExport writes, so a file from a newer build was sanitized into whatever this one understands and imported silently, changing the operator's criteria rather than saying the file was from a later version. Unsupported versions are now refused and reported. - the column tests used non-null assertions, which the frontend path instruction forbids and which hide a removed column behind an undefined access. A helper names the missing column instead. Co-Authored-By: Claude Opus 5 --- src/workspaces/event-log/evtx-columns.test.ts | 18 ++++++++- .../event-log/evtx-filter-store.test.ts | 29 ++++++++++---- src/workspaces/event-log/evtx-filter-store.ts | 7 +++- .../event-log/evtx-saved-filters.test.ts | 38 +++++++++++++++++++ .../event-log/evtx-saved-filters.ts | 15 +++++++- 5 files changed, 95 insertions(+), 12 deletions(-) diff --git a/src/workspaces/event-log/evtx-columns.test.ts b/src/workspaces/event-log/evtx-columns.test.ts index 05102f390..d5ecea6c8 100644 --- a/src/workspaces/event-log/evtx-columns.test.ts +++ b/src/workspaces/event-log/evtx-columns.test.ts @@ -10,12 +10,26 @@ import { EVTX_COLUMNS, moveColumn, sanitizeColumnConfig, + type EvtxColumnId, + type EvtxColumnSpec, toggleColumn, visibleColumns, type EvtxColumnConfig, } from "./evtx-columns"; import type { EvtxRecord } from "./types"; +/** + * The spec for `id`, failing with the id when it is absent. + * + * A non-null assertion would hide a removed column behind a confusing undefined access; this says + * which column went missing. + */ +function columnSpec(id: EvtxColumnId): EvtxColumnSpec { + const found = EVTX_COLUMNS.find((column) => column.id === id); + if (!found) throw new Error(`no column spec for ${id}`); + return found; +} + const config = (order: string[], widths = {}): EvtxColumnConfig => sanitizeColumnConfig({ order, widths }); @@ -77,13 +91,13 @@ describe("visibleColumns", () => { describe("columnWidth", () => { it("prefers an override over the default", () => { - const level = EVTX_COLUMNS.find((c) => c.id === "level")!; + const level = columnSpec("level"); expect(columnWidth(config(["level"], { level: 88 }), level)).toBe(88); expect(columnWidth(config(["level"]), level)).toBe(level.defaultWidth); }); it("keeps the description column unbounded", () => { - const message = EVTX_COLUMNS.find((c) => c.id === "message")!; + const message = columnSpec("message"); expect(columnWidth(config(["message"]), message)).toBeNull(); }); }); diff --git a/src/workspaces/event-log/evtx-filter-store.test.ts b/src/workspaces/event-log/evtx-filter-store.test.ts index 3e3bba7eb..a77ab6424 100644 --- a/src/workspaces/event-log/evtx-filter-store.test.ts +++ b/src/workspaces/event-log/evtx-filter-store.test.ts @@ -4,6 +4,13 @@ import { sanitizeCriteria } from "./evtx-saved-filters"; const criteria = () => sanitizeCriteria({ levels: ["Error"], search: "boot" }); +/** save() returns null for an empty name; every call here uses a real one, so assert that. */ +function saveNamed(name: string) { + const saved = useSavedFilterStore.getState().save(name, criteria()); + if (!saved) throw new Error(`save refused the name ${JSON.stringify(name)}`); + return saved; +} + beforeEach(() => { useSavedFilterStore.setState({ savedFilters: [] }); localStorage.clear(); @@ -11,18 +18,18 @@ beforeEach(() => { describe("useSavedFilterStore", () => { it("saves a filter and stamps it as used", () => { - const saved = useSavedFilterStore.getState().save("Boot errors", criteria()); + const saved = saveNamed("Boot errors"); expect(saved.name).toBe("Boot errors"); expect(saved.lastUsed).not.toBeNull(); expect(useSavedFilterStore.getState().savedFilters).toHaveLength(1); }); it("saving under an existing name updates rather than duplicating", () => { - const store = useSavedFilterStore.getState(); - const first = store.save("Boot", criteria()); + const first = saveNamed("Boot"); const second = useSavedFilterStore .getState() .save("boot", sanitizeCriteria({ search: "changed" })); + if (!second) throw new Error("save refused a valid name"); expect(useSavedFilterStore.getState().savedFilters).toHaveLength(1); expect(second.id).toBe(first.id); @@ -30,14 +37,14 @@ describe("useSavedFilterStore", () => { }); it("preserves the favorite flag when re-saving", () => { - const saved = useSavedFilterStore.getState().save("Boot", criteria()); + const saved = saveNamed("Boot"); useSavedFilterStore.getState().toggleFavorite(saved.id); useSavedFilterStore.getState().save("Boot", sanitizeCriteria({ search: "again" })); expect(useSavedFilterStore.getState().savedFilters[0].favorite).toBe(true); }); it("removes by id", () => { - const saved = useSavedFilterStore.getState().save("Boot", criteria()); + const saved = saveNamed("Boot"); useSavedFilterStore.getState().remove(saved.id); expect(useSavedFilterStore.getState().savedFilters).toEqual([]); }); @@ -46,9 +53,8 @@ describe("useSavedFilterStore", () => { // Favourite the one that loses on every other rule. Favouriting "Alpha" proved nothing: both // saves stamp lastUsed from the same clock tick, so the ordering fell through to the name // comparison and "Alpha" came first whether or not toggleFavorite did anything at all. - const store = useSavedFilterStore.getState(); - const zulu = store.save("Zulu", criteria()); - useSavedFilterStore.getState().save("Alpha", criteria()); + const zulu = saveNamed("Zulu"); + saveNamed("Alpha"); expect(useSavedFilterStore.getState().ordered()[0].name).toBe("Alpha"); @@ -70,4 +76,11 @@ describe("useSavedFilterStore", () => { ); expect(merged.savedFilters).toHaveLength(1); }); + + it("refuses a whitespace-only name instead of storing one that vanishes", () => { + // sanitizeSavedFilter drops an empty name on rehydration, so storing it would show the filter + // in the list and then lose it on restart, which reads as the app losing the operator's work. + expect(useSavedFilterStore.getState().save(" ", criteria())).toBeNull(); + expect(useSavedFilterStore.getState().savedFilters).toHaveLength(0); + }); }); diff --git a/src/workspaces/event-log/evtx-filter-store.ts b/src/workspaces/event-log/evtx-filter-store.ts index f27982847..2cca27505 100644 --- a/src/workspaces/event-log/evtx-filter-store.ts +++ b/src/workspaces/event-log/evtx-filter-store.ts @@ -18,7 +18,8 @@ import { interface SavedFilterState { savedFilters: EvtxSavedFilter[]; - save: (name: string, criteria: EvtxFilterCriteria) => EvtxSavedFilter; + /** Returns the stored filter, or null when the name is empty once trimmed. */ + save: (name: string, criteria: EvtxFilterCriteria) => EvtxSavedFilter | null; remove: (id: string) => void; toggleFavorite: (id: string) => void; markUsed: (id: string) => void; @@ -41,6 +42,10 @@ export const useSavedFilterStore = create()( save: (name, criteria) => { const trimmed = name.trim(); + // A whitespace-only name is refused rather than stored. sanitizeSavedFilter drops it on + // rehydration, so it would save, appear in the list, and then vanish on restart, which + // reads as the app losing the operator's filter. + if (!trimmed) return null; const existing = get().savedFilters.find( (filter) => filter.name.toLowerCase() === trimmed.toLowerCase() ); diff --git a/src/workspaces/event-log/evtx-saved-filters.test.ts b/src/workspaces/event-log/evtx-saved-filters.test.ts index a2ebb7d10..fcdb68da7 100644 --- a/src/workspaces/event-log/evtx-saved-filters.test.ts +++ b/src/workspaces/event-log/evtx-saved-filters.test.ts @@ -4,6 +4,7 @@ import { mergeFilters, orderFilters, parseFilterExport, + SAVED_FILTER_SCHEMA, sanitizeCriteria, sanitizeSavedFilter, type EvtxSavedFilter, @@ -144,3 +145,40 @@ describe("orderFilters", () => { expect(input.map((f) => f.name)).toEqual(["B", "A"]); }); }); + +describe("hostile stored values", () => { + it("rejects a non-finite lastUsed", () => { + // JSON admits 1e309, which parses to Infinity and would reach the ordering comparator as a + // non-finite operand. + const parsed = parseFilterExport( + JSON.stringify({ schema: 1, filters: [{ id: "a", name: "A", lastUsed: 1e309 }] }) + ); + expect(parsed.filters).toHaveLength(1); + expect(parsed.filters[0].lastUsed).toBeNull(); + }); + + it("keeps a finite lastUsed", () => { + const parsed = parseFilterExport( + JSON.stringify({ schema: 1, filters: [{ id: "a", name: "A", lastUsed: 1700000000000 }] }) + ); + expect(parsed.filters[0].lastUsed).toBe(1700000000000); + }); + + it("refuses an export written by a newer schema", () => { + // Sanitizing it into whatever this build understands would import silently and quietly change + // the operator's criteria rather than saying the file is from a later version. + const parsed = parseFilterExport( + JSON.stringify({ schema: 99, filters: [{ id: "a", name: "A" }] }) + ); + expect(parsed.filters).toHaveLength(0); + expect(parsed.unsupportedSchema).toBe(true); + }); + + it("still accepts the current schema", () => { + const parsed = parseFilterExport( + JSON.stringify({ schema: SAVED_FILTER_SCHEMA, filters: [{ id: "a", name: "A" }] }) + ); + expect(parsed.filters).toHaveLength(1); + expect(parsed.unsupportedSchema).toBeUndefined(); + }); +}); diff --git a/src/workspaces/event-log/evtx-saved-filters.ts b/src/workspaces/event-log/evtx-saved-filters.ts index 52a8a3eca..26be3e144 100644 --- a/src/workspaces/event-log/evtx-saved-filters.ts +++ b/src/workspaces/event-log/evtx-saved-filters.ts @@ -111,7 +111,11 @@ export function sanitizeSavedFilter(input: unknown, fallbackId: string): EvtxSav ) : [], criteria: sanitizeCriteria(input.criteria), - lastUsed: typeof input.lastUsed === "number" ? input.lastUsed : null, + // Finite only. JSON admits 1e309, which parses to Infinity and would reach the ordering + // comparator as a non-finite operand. + lastUsed: typeof input.lastUsed === "number" && Number.isFinite(input.lastUsed) + ? input.lastUsed + : null, }; } @@ -124,6 +128,8 @@ export function sanitizeSavedFilter(input: unknown, fallbackId: string): EvtxSav export function parseFilterExport(text: string): { filters: EvtxSavedFilter[]; skipped: number; + /** True when the file was written by a build using a schema this one does not know. */ + unsupportedSchema?: boolean; } { let parsed: unknown; try { @@ -132,6 +138,13 @@ export function parseFilterExport(text: string): { return { filters: [], skipped: 0 }; } + // The schema is written on export and must be checked on import. A newer file would otherwise be + // sanitized into whatever this build understands and imported silently, quietly changing the + // operator's criteria rather than telling them the file is from a later version. + if (isRecord(parsed) && parsed.schema !== undefined && parsed.schema !== SAVED_FILTER_SCHEMA) { + return { filters: [], skipped: 0, unsupportedSchema: true }; + } + const list = isRecord(parsed) && Array.isArray(parsed.filters) ? parsed.filters : []; const filters: EvtxSavedFilter[] = []; let skipped = 0; From a57ec6a095a7c10bd59d35775e64ee47db0d02d5 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 10 Aug 2026 10:17:54 -0400 Subject: [PATCH 57/85] fix(event-log): stop a wide id range hanging the tab, and name classic providers Findings from the unresolved CodeRabbit threads, which I had not been reading: earlier rounds only worked from each review's "Actionable comments" summary, so the accumulated open threads went unseen. Critical: the Event ID filter expanded ranges unbounded, on every keystroke. Typing 4624-46240000 built a set of tens of millions on the UI thread and froze the tab before the number was finished. Event IDs are 16 bits, so clamping to 65535 costs nothing an operator can use. Major: classic sources emit with no Name attribute, and only Name was read. Every classic event therefore carried provider "Unknown", which meant no map could match it and no description could render for it. Name still wins where both are present. Also: - the XML export concatenated each record's own declaration into the middle of the document, which is legal only at the very start. The output was a file no XML parser would open. - provider payloads inflated with no size cap. These databases are evidence supplied by someone else, so a small blob could expand until memory ran out before serde_json saw it. Capped at 64 MB, far above any real payload. - toolbar font sizes were hardcoded at 11px, which the workspace path instruction forbids: raising the list font left these controls small. Derived from logListFontSize now. - two assertions in the gated integration test were not invariants, against that file's own stated rule. Event ID 0 is legal and in-box providers emit it, and a capture past the reader's per-file cap correctly reports being truncated. - a real machine name survived in a frontend fixture after the Rust ones were replaced. Co-Authored-By: Claude Opus 5 --- src-tauri/src/event_log/event_node.rs | 32 +++++++++++- src-tauri/src/event_log/export.rs | 50 +++++++++++++++++-- src-tauri/src/event_log/provider_db.rs | 16 +++++- src-tauri/tests/event_log_real_evtx.rs | 18 ++++--- src-tauri/tests/graph_esp_diagnostics.rs | 7 ++- src-tauri/tests/jamf_environment.rs | 5 +- src-tauri/tests/jamf_ipc_contract.rs | 5 +- src-tauri/tests/jamf_known_sources.rs | 15 +++--- src-tauri/tests/jamf_parser_robustness.rs | 25 +++++++--- src-tauri/tests/jamf_policy_log_parsing.rs | 11 ++-- src-tauri/tests/jamf_real_fixtures.rs | 9 ++-- .../tests/jamf_self_service_log_parsing.rs | 10 +++- src/workspaces/event-log/EvtxFilterBar.tsx | 15 ++++-- src/workspaces/event-log/evtx-columns.test.ts | 2 +- src/workspaces/event-log/evtx-filter.test.ts | 24 +++++++++ src/workspaces/event-log/evtx-filter.ts | 15 +++++- 16 files changed, 219 insertions(+), 40 deletions(-) diff --git a/src-tauri/src/event_log/event_node.rs b/src-tauri/src/event_log/event_node.rs index 0f50103fc..b765218ba 100644 --- a/src-tauri/src/event_log/event_node.rs +++ b/src-tauri/src/event_log/event_node.rs @@ -183,7 +183,12 @@ pub fn extract_system_fields(root: &EventNode) -> SystemFields { }; SystemFields { - provider: attribute_of("Provider", "Name"), + // Manifest providers write Name; classic sources write only EventSourceName, for example + // . Reading just Name left every classic + // event with provider "Unknown", which meant no map could match it and no description + // could be rendered for it. + provider: attribute_of("Provider", "Name") + .or_else(|| attribute_of("Provider", "EventSourceName")), // Classic providers write `1000`. The id is the // element text in both shapes; the qualifier is separate and not part of the id. event_id: text_of("EventID").and_then(|value| value.parse().ok()), @@ -379,4 +384,29 @@ mod tests { assert!(parse_event_xml("").is_err() || parse_event_xml(" with no Name is what classic sources emit. Reading + // only Name left these as "Unknown", so no map matched and no description rendered. + let xml = r#" + + 1000 + "#; + let fields = extract_system_fields(&parse_event_xml(xml).expect("parses")); + assert_eq!(fields.provider.as_deref(), Some("Application Error")); + } + + #[test] + fn a_manifest_provider_still_wins_on_name() { + // Some events carry both; Name is the modern identity and must take precedence. + let xml = r#" + + "#; + let fields = extract_system_fields(&parse_event_xml(xml).expect("parses")); + assert_eq!( + fields.provider.as_deref(), + Some("Microsoft-Windows-Kernel-General") + ); + } } diff --git a/src-tauri/src/event_log/export.rs b/src-tauri/src/event_log/export.rs index 5ffba6dd8..609f39717 100644 --- a/src-tauri/src/event_log/export.rs +++ b/src-tauri/src/event_log/export.rs @@ -93,6 +93,22 @@ fn escape_delimited(value: &str, delimiter: char) -> String { } } +/// Removes a leading `` declaration. +/// +/// Returns the input unchanged when there is none, and leaves everything after the declaration +/// exactly as the source wrote it. +fn strip_xml_declaration(xml: &str) -> &str { + let trimmed = xml.trim_start(); + let Some(rest) = trimmed.strip_prefix("") { + Some(end) => rest[end + 2..].trim_start(), + // A declaration that never closes is not something to silently repair. + None => xml, + } +} + fn optional(value: Option) -> String { value.map(|v| v.to_string()).unwrap_or_default() } @@ -147,9 +163,12 @@ pub fn export_records(records: &[EvtxRecord], format: ExportFormat) -> Result { let mut out = String::from("\n\n"); for record in records { - // The provider's own XML is passed through untouched. Re-encoding it would risk - // changing what the source actually said, which matters when an export is evidence. - out.push_str(record.raw_xml.trim()); + // The provider's own XML is passed through otherwise untouched: re-encoding it + // would risk changing what the source actually said, which matters when an export + // is evidence. Only the per-record declaration is removed, because the evtx reader + // prefixes every record with one and a declaration is legal only at the very start + // of a document. Concatenating them produced a file no XML parser would open. + out.push_str(strip_xml_declaration(record.raw_xml.trim())); out.push('\n'); } out.push_str("\n"); @@ -276,6 +295,31 @@ mod tests { assert!(out.trim_end().ends_with("")); } + #[test] + fn per_record_xml_declarations_are_stripped_from_the_concatenation() { + // The evtx reader prefixes every record with a declaration, and a declaration is legal + // only at the very start of a document. Concatenating them produced a file no XML parser + // would open. + let mut record = record("x"); + record.raw_xml = + "\n".into(); + let out = export_records(&[record.clone(), record], ExportFormat::Xml).expect("exports"); + + assert_eq!( + out.matches("")); + assert_eq!(out.matches("").count(), 2, "both records survive"); + } + + #[test] + fn a_record_without_a_declaration_is_untouched() { + let out = export_records(&[record("x")], ExportFormat::Xml).expect("exports"); + assert!(out.contains("")); + } + #[test] fn an_empty_export_is_still_well_formed() { assert!(export_records(&[], ExportFormat::Json).expect("exports") == "[]"); diff --git a/src-tauri/src/event_log/provider_db.rs b/src-tauri/src/event_log/provider_db.rs index ceb92e4c2..f138c1428 100644 --- a/src-tauri/src/event_log/provider_db.rs +++ b/src-tauri/src/event_log/provider_db.rs @@ -45,15 +45,29 @@ pub struct ProviderDbInfo { /// /// An empty BLOB is a legitimately empty section rather than a fault, so it deserializes to the /// type's default instead of erroring. +/// Largest decompressed provider payload accepted from a database. +/// +/// The biggest real provider in a 15.8 MB capture inflates to well under a megabyte, so 64 MB +/// refuses only what could not be a genuine payload. +const MAX_PROVIDER_PAYLOAD_BYTES: u64 = 64 * 1024 * 1024; + fn inflate_json(blob: &[u8]) -> Result { if blob.is_empty() { return Ok(T::default()); } - let mut decoder = GzDecoder::new(blob); + // Capped. These databases are evidence supplied by someone else, and an unbounded inflate lets + // a small blob expand to gigabytes and exhaust memory before serde_json ever sees it. The cap + // is far above any real provider payload, so it refuses only what could not be genuine. + let mut decoder = GzDecoder::new(blob).take(MAX_PROVIDER_PAYLOAD_BYTES + 1); let mut json = String::new(); decoder .read_to_string(&mut json) .map_err(|error| format!("provider payload is not valid gzip: {error}"))?; + if json.len() as u64 > MAX_PROVIDER_PAYLOAD_BYTES { + return Err(format!( + "provider payload inflates past {MAX_PROVIDER_PAYLOAD_BYTES} bytes and was refused" + )); + } if json.trim().is_empty() { return Ok(T::default()); } diff --git a/src-tauri/tests/event_log_real_evtx.rs b/src-tauri/tests/event_log_real_evtx.rs index 509bd5869..0033c7034 100644 --- a/src-tauri/tests/event_log_real_evtx.rs +++ b/src-tauri/tests/event_log_real_evtx.rs @@ -89,7 +89,7 @@ fn identity_is_populated_rather_than_defaulted() { "record {} has no channel", record.event_record_id ); - assert_ne!(record.event_id, 0, "record has no event id"); + // Event ID 0 is legal and several in-box providers emit it, so it is not an invariant. assert_ne!( record.timestamp_epoch, 0, "record {} sorted to the epoch, so the timeline order is wrong", @@ -143,12 +143,18 @@ fn the_xml_export_emits_the_provider_representation() { #[test] fn a_healthy_capture_reports_no_gaps() { let Some(result) = parsed() else { return }; - // The messages are a gap report. A clean file producing one would train an operator to ignore - // them, which is how a real gap goes unnoticed. + // A gap report on a clean file would train an operator to ignore it, which is how a real gap + // goes unnoticed. Not an unconditional invariant, though: a capture larger than the reader's + // per-file cap legitimately reports being truncated, and that message is correct rather than a + // false alarm. Anything else on a healthy file is not. + let unexpected: Vec<&String> = result + .error_messages + .iter() + .filter(|message| !message.contains("stopped at")) + .collect(); assert!( - result.error_messages.is_empty(), - "clean capture reported gaps: {:?}", - result.error_messages + unexpected.is_empty(), + "clean capture reported gaps: {unexpected:?}" ); assert_eq!(result.total_records, result.records.len() as u64); } diff --git a/src-tauri/tests/graph_esp_diagnostics.rs b/src-tauri/tests/graph_esp_diagnostics.rs index afd08dfd2..541e60deb 100644 --- a/src-tauri/tests/graph_esp_diagnostics.rs +++ b/src-tauri/tests/graph_esp_diagnostics.rs @@ -228,7 +228,12 @@ fn graph_permission_upgrade_command_runs_owned_wam_on_a_blocking_worker() { "pub(crate) fn authenticate(", "pub(crate) fn request_missing_permissions(", ); - assert_eq!(authenticate.matches("wam::authentication_deadline()").count(), 1); + assert_eq!( + authenticate + .matches("wam::authentication_deadline()") + .count(), + 1 + ); assert!(authenticate.contains("match wam::acquire_token(hwnd_raw, deadline, lease)")); assert!(authenticate.contains("probe_host_capability_for_authentication(deadline, lease)")); assert!(graph_source.contains("std::time::Duration::from_secs(120)")); diff --git a/src-tauri/tests/jamf_environment.rs b/src-tauri/tests/jamf_environment.rs index e66b50f8d..6fb11c576 100644 --- a/src-tauri/tests/jamf_environment.rs +++ b/src-tauri/tests/jamf_environment.rs @@ -9,7 +9,10 @@ fn collect_environment_does_not_panic() { // An installed binary does not guarantee `jamf version` succeeds (it can // fail or time out), so only the inverse is a real invariant. if !env.jamf_installed { - assert!(env.jamf_version.is_none(), "version reported without a binary"); + assert!( + env.jamf_version.is_none(), + "version reported without a binary" + ); } } diff --git a/src-tauri/tests/jamf_ipc_contract.rs b/src-tauri/tests/jamf_ipc_contract.rs index 53d41267c..4a356e1f6 100644 --- a/src-tauri/tests/jamf_ipc_contract.rs +++ b/src-tauri/tests/jamf_ipc_contract.rs @@ -45,7 +45,10 @@ fn policy_trigger_data_variants_carry_a_value_field() { #[test] fn policy_result_matches_the_typescript_union() { assert_eq!(json(&JamfPolicyResult::Success), r#"{"type":"success"}"#); - assert_eq!(json(&JamfPolicyResult::InProgress), r#"{"type":"inProgress"}"#); + assert_eq!( + json(&JamfPolicyResult::InProgress), + r#"{"type":"inProgress"}"# + ); assert_eq!(json(&JamfPolicyResult::Unknown), r#"{"type":"unknown"}"#); assert_eq!( json(&JamfPolicyResult::Failure("Error running recon".into())), diff --git a/src-tauri/tests/jamf_known_sources.rs b/src-tauri/tests/jamf_known_sources.rs index e770c9979..4c43778d0 100644 --- a/src-tauri/tests/jamf_known_sources.rs +++ b/src-tauri/tests/jamf_known_sources.rs @@ -22,7 +22,10 @@ fn jamf_known_sources_present() { let all = build_known_log_sources(); let jamf = jamf_sources(&all); let ids: Vec<&str> = jamf.iter().map(|s| s.id.as_str()).collect(); - assert!(ids.contains(&"macos-jamf-log"), "missing macos-jamf-log: {ids:?}"); + assert!( + ids.contains(&"macos-jamf-log"), + "missing macos-jamf-log: {ids:?}" + ); assert!(ids.contains(&"macos-jamf-app-support-logs")); assert!(ids.contains(&"macos-jamf-receipts")); assert!(ids.contains(&"macos-jamf-self-service-log")); @@ -40,11 +43,11 @@ fn jamf_log_default_file_is_jamf_log() { .iter() .find(|s| s.id == "macos-jamf-log") .expect("macos-jamf-log should exist"); - let intent = entry.default_file_intent.as_ref().expect("should have default file intent"); - assert!(intent - .preferred_file_names - .iter() - .any(|n| n == "jamf.log")); + let intent = entry + .default_file_intent + .as_ref() + .expect("should have default file intent"); + assert!(intent.preferred_file_names.iter().any(|n| n == "jamf.log")); } #[cfg(not(target_os = "macos"))] diff --git a/src-tauri/tests/jamf_parser_robustness.rs b/src-tauri/tests/jamf_parser_robustness.rs index 7bed41c27..1bba452c9 100644 --- a/src-tauri/tests/jamf_parser_robustness.rs +++ b/src-tauri/tests/jamf_parser_robustness.rs @@ -32,7 +32,11 @@ fn policy_log_survives_invalid_utf8_and_keeps_all_lines() { let result = parse_policy_log_impl(&path).expect("must not fail on invalid UTF-8"); assert_eq!(result.total_lines, 3); - assert_eq!(result.events.len(), 3, "every line must still yield an event"); + assert_eq!( + result.events.len(), + 3, + "every line must still yield an event" + ); assert_eq!(result.unparsed_lines, 0); let names: Vec<&str> = result @@ -42,7 +46,11 @@ fn policy_log_survives_invalid_utf8_and_keeps_all_lines() { .collect(); assert_eq!(names[0], "First"); assert_eq!(names[2], "Third"); - assert!(names[1].starts_with("Ba"), "decoded lossily: {:?}", names[1]); + assert!( + names[1].starts_with("Ba"), + "decoded lossily: {:?}", + names[1] + ); let _ = std::fs::remove_file(&path); } @@ -77,7 +85,10 @@ fn policy_log_tolerates_a_truncated_final_line() { let result = parse_policy_log_impl(&path).expect("parse"); assert_eq!(result.total_lines, 2); assert_eq!(result.events.len(), 1); - assert_eq!(result.unparsed_lines, 1, "the partial line is counted, not fatal"); + assert_eq!( + result.unparsed_lines, 1, + "the partial line is counted, not fatal" + ); let _ = std::fs::remove_file(&path); } @@ -190,9 +201,11 @@ fn an_over_long_line_does_not_disturb_later_offsets() { fn missing_files_are_not_errors_except_for_the_policy_log() { // Self Service / Connect may legitimately be absent; jamf.log going missing // is a condition worth surfacing. - assert!(parse_self_service_log_impl(Path::new("/nonexistent/ss.log")) - .expect("absent Self Service log is empty, not an error") - .is_empty()); + assert!( + parse_self_service_log_impl(Path::new("/nonexistent/ss.log")) + .expect("absent Self Service log is empty, not an error") + .is_empty() + ); assert!(parse_connect_log_impl(Path::new("/nonexistent/jc.log")) .expect("absent Connect log is empty, not an error") .is_empty()); diff --git a/src-tauri/tests/jamf_policy_log_parsing.rs b/src-tauri/tests/jamf_policy_log_parsing.rs index fb1c33d53..aac7fb35d 100644 --- a/src-tauri/tests/jamf_policy_log_parsing.rs +++ b/src-tauri/tests/jamf_policy_log_parsing.rs @@ -77,7 +77,10 @@ fn classifies_jss_connectivity_failure() { #[test] fn unparsed_lines_counted() { let result = parse_policy_log_impl(Path::new(BASIC)).expect("parse"); - assert_eq!(result.total_lines, result.events.len() + result.unparsed_lines); + assert_eq!( + result.total_lines, + result.events.len() + result.unparsed_lines + ); } #[test] @@ -129,8 +132,10 @@ Wed Jul 22 20:10:32 host jamf[6003]: Successfully installed Zscaler-osx-4.5.2.31 let installing = result .events .iter() - .find(|e| matches!(&e.result, JamfPolicyResult::InProgress) - && e.policy_name.as_deref() == Some("Zscaler-osx-4.5.2.312-installer.pkg")) + .find(|e| { + matches!(&e.result, JamfPolicyResult::InProgress) + && e.policy_name.as_deref() == Some("Zscaler-osx-4.5.2.312-installer.pkg") + }) .expect("the Installing line should name the package"); assert!(matches!(&installing.trigger, JamfPolicyTrigger::Other(k) if k == "install")); diff --git a/src-tauri/tests/jamf_real_fixtures.rs b/src-tauri/tests/jamf_real_fixtures.rs index 575b90168..efe9bd60d 100644 --- a/src-tauri/tests/jamf_real_fixtures.rs +++ b/src-tauri/tests/jamf_real_fixtures.rs @@ -117,7 +117,10 @@ fn self_service_log_yields_user_actions() { let Some(dir) = fixture_dir() else { return }; let events = parse_self_service_log_impl(&dir.join("logs/selfservice.log")).expect("parse"); - assert!(!events.is_empty(), "real selfservice.log produced no events"); + assert!( + !events.is_empty(), + "real selfservice.log produced no events" + ); assert!( events.iter().any(|e| e.action == "triggerPolicy"), "capture is known to contain Self Service-initiated installs" @@ -170,8 +173,8 @@ fn captured_profiles_xml_parses_and_filters_to_jamf() { profiles.len() ); - let filtered = app_lib::jamf::profiles::filter_jamf_profiles_impl(profiles, None) - .expect("filter"); + let filtered = + app_lib::jamf::profiles::filter_jamf_profiles_impl(profiles, None).expect("filter"); assert!( !filtered.profiles.is_empty(), "payload-prefix matching found no JAMF profiles in a JAMF capture" diff --git a/src-tauri/tests/jamf_self_service_log_parsing.rs b/src-tauri/tests/jamf_self_service_log_parsing.rs index ada433488..b33283c99 100644 --- a/src-tauri/tests/jamf_self_service_log_parsing.rs +++ b/src-tauri/tests/jamf_self_service_log_parsing.rs @@ -28,7 +28,10 @@ fn parses_real_selfservice_log_shapes() { // API chatter is one action with the endpoint as the item. assert_eq!(events[4].action, "request"); - assert_eq!(events[4].item_name.as_deref(), Some("updateDevicePushToken")); + assert_eq!( + events[4].item_name.as_deref(), + Some("updateDevicePushToken") + ); assert_eq!(events[7].action, "warning"); assert!(events[7] @@ -37,7 +40,10 @@ fn parses_real_selfservice_log_shapes() { .is_some_and(|m| m.starts_with("A customized icon"))); // Binary requests are the user-side operations. - let triggers: Vec<&_> = events.iter().filter(|e| e.action == "triggerPolicy").collect(); + let triggers: Vec<&_> = events + .iter() + .filter(|e| e.action == "triggerPolicy") + .collect(); assert_eq!(triggers.len(), 2); assert_eq!(events.iter().filter(|e| e.action == "doRecon").count(), 1); } diff --git a/src/workspaces/event-log/EvtxFilterBar.tsx b/src/workspaces/event-log/EvtxFilterBar.tsx index a2b3a6455..e0202b797 100644 --- a/src/workspaces/event-log/EvtxFilterBar.tsx +++ b/src/workspaces/event-log/EvtxFilterBar.tsx @@ -5,6 +5,8 @@ import { save } from "@tauri-apps/plugin-dialog"; import { selectVisibleRecords, EVTX_GROUP_LABELS, type EvtxGroupField } from "./evtx-filter"; import { useSavedFilterStore } from "./evtx-filter-store"; import { orderFilters, sanitizeCriteria } from "./evtx-saved-filters"; +import { getLogListMetrics } from "../../lib/log-accessibility"; +import { useUiStore } from "../../stores/ui-store"; import { availableColumns, discoverMappedProperties, @@ -70,6 +72,11 @@ export function EvtxFilterBar() { const setSortDirection = useEvtxStore((s) => s.setSortDirection); const timeWindow = useEvtxStore((s) => s.timeWindow); const timeZoneMode = useEvtxStore((s) => s.timeZoneMode); + // Sized from the operator's list font rather than hardcoded, so raising the list size raises + // these controls with it. Clamped down a step because a toolbar control sits beside the list + // rather than in it. + const logListFontSize = useUiStore((s) => s.logListFontSize); + const controlFontSize = `${Math.max(11, getLogListMetrics(logListFontSize).fontSize - 1)}px`; const records = useEvtxStore((s) => s.records); // Map columns are offered only when a loaded map actually produced them, so the chooser does not // fill with columns that are empty for the log in front of the operator. @@ -223,7 +230,7 @@ export function EvtxFilterBar() { size="small" appearance="outline" onClick={() => setTimeZoneMode(timeZoneMode === "local" ? "utc" : "local")} - style={{ minWidth: "auto", padding: "2px 8px", fontSize: "11px" }} + style={{ minWidth: "auto", padding: "2px 8px", fontSize: controlFontSize }} title={ timeZoneMode === "utc" ? "Event times are shown in UTC, as Windows recorded them. Click for local time." @@ -252,7 +259,7 @@ export function EvtxFilterBar() { style={{ minWidth: "auto", padding: "2px 8px", - fontSize: "11px", + fontSize: controlFontSize, borderColor: active ? undefined : LEVEL_COLORS[level], color: active ? undefined : LEVEL_COLORS[level], }} @@ -405,7 +412,7 @@ export function EvtxFilterBar() { {exportState && ( - + {exportState} )} @@ -423,7 +430,7 @@ export function EvtxFilterBar() {
diff --git a/src/workspaces/event-log/evtx-columns.test.ts b/src/workspaces/event-log/evtx-columns.test.ts index d5ecea6c8..d0696864a 100644 --- a/src/workspaces/event-log/evtx-columns.test.ts +++ b/src/workspaces/event-log/evtx-columns.test.ts @@ -43,7 +43,7 @@ function record(partial: Partial = {}): EvtxRecord { channel: "Application", eventId: 326, level: "Error", - computer: "RING0IVY24-01", + computer: "TESTHOST-01", message: "something happened", eventData: [], rawXml: "", diff --git a/src/workspaces/event-log/evtx-filter.test.ts b/src/workspaces/event-log/evtx-filter.test.ts index 7b225cf35..e03b28fdf 100644 --- a/src/workspaces/event-log/evtx-filter.test.ts +++ b/src/workspaces/event-log/evtx-filter.test.ts @@ -37,3 +37,27 @@ describe("parseEventIdFilter", () => { expect(parseEventIdFilter("abc, def")).toBeNull(); }); }); + +describe("event id range bounds", () => { + it("does not expand past the 16-bit event id space", () => { + // Typed on every keystroke. Unbounded, "4624-46240000" builds a set of tens of millions on the + // UI thread and the tab stops responding before the operator finishes typing. + const started = Date.now(); + const ids = parseEventIdFilter("4624-46240000"); + expect(Date.now() - started).toBeLessThan(1000); + expect(ids).not.toBeNull(); + expect(ids!.size).toBeLessThanOrEqual(65536); + expect(ids!.has(4624)).toBe(true); + expect(ids!.has(65535)).toBe(true); + expect(ids!.has(65536)).toBe(false); + }); + + it("yields nothing for a range entirely above the id space", () => { + expect(parseEventIdFilter("100000-200000")).toBeNull(); + }); + + it("still expands an ordinary range", () => { + const ids = parseEventIdFilter("4624-4626"); + expect([...ids!].sort((a, b) => a - b)).toEqual([4624, 4625, 4626]); + }); +}); diff --git a/src/workspaces/event-log/evtx-filter.ts b/src/workspaces/event-log/evtx-filter.ts index 5c71fd147..c962ecd5c 100644 --- a/src/workspaces/event-log/evtx-filter.ts +++ b/src/workspaces/event-log/evtx-filter.ts @@ -24,7 +24,11 @@ export function parseEventIdFilter(raw: string): Set | null { const low = Number(range[1]); const high = Number(range[2]); const [from, to] = low <= high ? [low, high] : [high, low]; - for (let id = from; id <= to; id += 1) ids.add(id); + // Clamped to the range an Event ID can actually occupy. This runs on every keystroke, so + // "4624-46240000" typed halfway would otherwise build a set of tens of millions on the UI + // thread and freeze the tab before the operator finished the number. + if (from > MAX_EVENT_ID) continue; + for (let id = from; id <= Math.min(to, MAX_EVENT_ID); id += 1) ids.add(id); continue; } const single = Number(token); @@ -33,6 +37,15 @@ export function parseEventIdFilter(raw: string): Set | null { return ids.size > 0 ? ids : null; } +/** + * Largest value a Windows Event ID can hold. + * + * The field is 16 bits, so nothing above this can match an event and expanding past it only costs + * time. Used to bound range expansion rather than to reject input: an operator mid-way through + * typing a number should see no result, not an error. + */ +const MAX_EVENT_ID = 65535; + /** The subset of store state that decides which records are on screen. */ export interface VisibleRecordsInput { records: EvtxRecord[]; From 9133042d2fd009627e95bf361b2303fe86ed56cc Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 10 Aug 2026 11:27:43 -0400 Subject: [PATCH 58/85] fix(event-log): export the columns the grid showed, and stop reopening databases More findings from the unresolved thread backlog. Delimited export dropped two things the grid displays. User SID is a primary pivot for triage and was never a column at all; map-derived values were not either, so exporting the grid gave an analyst fewer columns than they had been looking at. Both are exported now: User SID as a fixed column, and the union of map properties appended after it so a record the map did not match still lines up with the header. An incomplete mapping exports empty for the same reason it renders empty, rather than putting a literal %3 into a cell. export_records rendered every record into one String on a tokio worker while every other heavy command in the file used spawn_blocking. The XML format concatenates raw_xml for up to a hundred thousand records, so it could hold a runtime worker for seconds and stall unrelated IPC. The provider store reopened every registered database on each cache miss, and opening also runs a schema probe, so a miss cost an open plus a probe per database for every distinct provider name in a file. Databases are opened once at registration and held. The doc claiming &mut self was left over from before the cache moved behind its own lock. Co-Authored-By: Claude Opus 5 --- src-tauri/src/event_log/commands.rs | 12 ++- src-tauri/src/event_log/export.rs | 112 +++++++++++++++++++++++-- src-tauri/src/event_log/provider_db.rs | 26 ++++-- 3 files changed, 132 insertions(+), 18 deletions(-) diff --git a/src-tauri/src/event_log/commands.rs b/src-tauri/src/event_log/commands.rs index 7d15bd12c..c5bafea8b 100644 --- a/src-tauri/src/event_log/commands.rs +++ b/src-tauri/src/event_log/commands.rs @@ -174,14 +174,20 @@ pub async fn evtx_export_records( format: super::export::ExportFormat, destination: String, ) -> Result { - let rendered = super::export::export_records(&records, format)?; + let record_count = records.len(); + // Rendered on a blocking thread, like every other heavy command here. The XML format + // concatenates raw_xml for up to a hundred thousand records into one String, which can occupy + // a runtime worker for seconds and stall unrelated IPC. + let rendered = + tokio::task::spawn_blocking(move || super::export::export_records(&records, format)) + .await + .map_err(|error| format!("export task failed: {error}"))??; let byte_count = rendered.len() as u64; tokio::fs::write(&destination, rendered) .await .map_err(|error| format!("cannot write {destination}: {error}"))?; log::info!( - "event=evtx_export destination=\"{destination}\" records={} bytes={byte_count}", - records.len() + "event=evtx_export destination=\"{destination}\" records={record_count} bytes={byte_count}" ); Ok(byte_count) } diff --git a/src-tauri/src/event_log/export.rs b/src-tauri/src/event_log/export.rs index 609f39717..de1cfcb71 100644 --- a/src-tauri/src/event_log/export.rs +++ b/src-tauri/src/event_log/export.rs @@ -45,7 +45,11 @@ impl ExportFormat { } } -const COLUMNS: [&str; 13] = [ +/// Columns every delimited export carries, in order. +/// +/// `User SID` is here because it is a primary pivot for event triage; leaving it out meant an +/// analyst who exported the grid got fewer columns than the grid had shown them. +const COLUMNS: [&str; 14] = [ "Event Time", "Record ID", "Event ID", @@ -57,10 +61,28 @@ const COLUMNS: [&str; 13] = [ "Opcode", "Process ID", "Thread ID", + "User SID", "Keywords", "Description", ]; +/// Map-derived column names present across `records`, in first-seen order. +/// +/// Appended after the fixed columns so a delimited export carries the same map values the grid +/// renders. Discovered from the records rather than declared, because which properties exist +/// depends on which maps matched. +fn mapped_columns(records: &[EvtxRecord]) -> Vec { + let mut names: Vec = Vec::new(); + for record in records { + for column in &record.mapped { + if !names.iter().any(|existing| existing == &column.property) { + names.push(column.property.clone()); + } + } + } + names +} + /// Neutralizes a value that a spreadsheet would otherwise execute as a formula. /// /// Excel and LibreOffice treat a leading `=`, `+`, `-` or `@` as the start of a formula. Event @@ -113,8 +135,8 @@ fn optional(value: Option) -> String { value.map(|v| v.to_string()).unwrap_or_default() } -fn row_of(record: &EvtxRecord) -> [String; 13] { - [ +fn row_of(record: &EvtxRecord, mapped: &[String]) -> Vec { + let mut row: Vec = vec![ record.timestamp.clone(), record.event_record_id.to_string(), record.event_id.to_string(), @@ -126,9 +148,23 @@ fn row_of(record: &EvtxRecord) -> [String; 13] { optional(record.opcode), optional(record.process_id), optional(record.thread_id), + record.user_sid.clone().unwrap_or_default(), record.keywords.clone().unwrap_or_default(), record.message.clone(), - ] + ]; + for property in mapped { + // An incomplete mapping renders empty here for the same reason it does in the grid: a + // half-substituted template would put a literal %3 into an exported cell. + let value = record + .mapped + .iter() + .find(|column| &column.property == property) + .filter(|column| column.complete) + .map(|column| column.text.clone()) + .unwrap_or_default(); + row.push(value); + } + row } /// Renders records in `format`. @@ -136,17 +172,20 @@ pub fn export_records(records: &[EvtxRecord], format: ExportFormat) -> Result { let delimiter = format.delimiter(); + let mapped = mapped_columns(records); let mut out = String::new(); out.push_str( &COLUMNS .iter() - .map(|column| escape_delimited(column, delimiter)) + .map(|column| column.to_string()) + .chain(mapped.iter().cloned()) + .map(|column| escape_delimited(&column, delimiter)) .collect::>() .join(&delimiter.to_string()), ); out.push('\n'); for record in records { - let row = row_of(record); + let row = row_of(record, &mapped); out.push_str( &row.iter() .map(|value| escape_delimited(value, delimiter)) @@ -269,6 +308,67 @@ mod tests { assert_eq!(fields[10], "", "thread id absent"); } + #[test] + fn the_user_sid_reaches_the_export() { + // A primary pivot for triage. Leaving it out gave an analyst fewer columns than the grid + // had shown them. + let out = export_records(&[record("x")], ExportFormat::Csv).expect("exports"); + assert!(out.lines().next().expect("header").contains("User SID")); + assert!(out.lines().nth(1).expect("row").contains("S-1-5-18")); + } + + #[test] + fn map_derived_columns_are_appended_after_the_fixed_ones() { + let mut mapped = record("x"); + mapped.mapped = vec![crate::event_log::maps::MappedColumn { + property: "PayloadData1".into(), + text: "cmd.exe".into(), + complete: true, + }]; + + let out = export_records(&[mapped], ExportFormat::Csv).expect("exports"); + let header = out.lines().next().expect("header"); + assert!(header.ends_with("PayloadData1"), "{header}"); + assert!(out.lines().nth(1).expect("row").ends_with("cmd.exe")); + } + + #[test] + fn a_record_without_a_mapped_value_leaves_the_cell_empty() { + // The union of properties is the header, so a record the map did not match still has to + // line up with it. + let mut mapped = record("has one"); + mapped.mapped = vec![crate::event_log::maps::MappedColumn { + property: "PayloadData1".into(), + text: "cmd.exe".into(), + complete: true, + }]; + let plain = record("has none"); + + let out = export_records(&[mapped, plain], ExportFormat::Csv).expect("exports"); + let rows: Vec<&str> = out.lines().collect(); + let columns = |line: &str| line.split(',').count(); + assert_eq!(columns(rows[0]), columns(rows[1])); + assert_eq!(columns(rows[0]), columns(rows[2])); + assert!( + rows[2].ends_with(','), + "the unmatched cell is empty: {}", + rows[2] + ); + } + + #[test] + fn an_incomplete_mapping_exports_empty_rather_than_a_raw_template() { + let mut mapped = record("x"); + mapped.mapped = vec![crate::event_log::maps::MappedColumn { + property: "PayloadData1".into(), + text: "ran %3 as adam".into(), + complete: false, + }]; + + let out = export_records(&[mapped], ExportFormat::Csv).expect("exports"); + assert!(!out.contains("%3"), "{out}"); + } + #[test] fn tsv_uses_tabs_and_quotes_values_containing_them() { let out = export_records(&[record("a\tb")], ExportFormat::Tsv).expect("exports"); diff --git a/src-tauri/src/event_log/provider_db.rs b/src-tauri/src/event_log/provider_db.rs index f138c1428..189908ec5 100644 --- a/src-tauri/src/event_log/provider_db.rs +++ b/src-tauri/src/event_log/provider_db.rs @@ -210,7 +210,6 @@ impl ProviderDb { /// outlived any workspace the operator closed, with no way to reset it. #[derive(Default)] pub struct ProviderStore { - databases: Vec, /// Lowercased provider name to metadata, including negative results so a provider absent from /// every database is not looked up again for every event that mentions it. /// @@ -221,6 +220,8 @@ pub struct ProviderStore { /// hold a write guard on the whole store for the length of a file, blocking every other reader /// just to populate a cache. cache: Mutex>>>, + /// The registered databases, opened once at registration and reused for every lookup. + open_databases: Mutex>, info: Vec, } @@ -234,7 +235,7 @@ impl ProviderStore { ) })?; - let mut databases = Vec::new(); + let mut databases: Vec = Vec::new(); let mut info = Vec::new(); let mut failures: Vec = Vec::new(); @@ -263,7 +264,9 @@ impl ProviderStore { match ProviderDb::open(&path) { Ok(database) => { info.push(database.info().clone()); - databases.push(path); + // Kept open. Reopening per lookup also re-ran the schema probe inside open(), + // once per registered database for every distinct provider name in a file. + databases.push(database); } // A file that is not a provider database is reported, not fatal: the directory is // user-supplied and may hold anything. @@ -271,7 +274,9 @@ impl ProviderStore { } } - self.databases = databases; + if let Ok(mut open) = self.open_databases.lock() { + *open = databases; + } self.info = info.clone(); if let Ok(mut cache) = self.cache.lock() { cache.clear(); @@ -293,9 +298,9 @@ impl ProviderStore { /// Metadata for `provider_name`, consulting registered databases in order and caching it. /// - /// Takes `&mut self` because a lookup populates the cache, including the negative result. That - /// is what stops a provider absent from every database being searched again for every event - /// that names it. + /// The cache is behind its own lock, so this needs only `&self`; a lookup still populates it, + /// including the negative result, which is what stops a provider absent from every database + /// being searched again for every event that names it. pub fn provider(&self, provider_name: &str) -> Option> { let key = provider_name.to_ascii_lowercase(); if let Ok(cache) = self.cache.lock() { @@ -304,9 +309,12 @@ impl ProviderStore { } } + // Databases are opened once and held, not reopened per lookup. Opening also runs a schema + // probe, so a miss used to pay an open plus that probe for every registered database, for + // every distinct provider name in the file. let mut found = None; - for path in &self.databases { - if let Ok(database) = ProviderDb::open(path) { + if let Ok(open) = self.open_databases.lock() { + for database in open.iter() { if let Ok(Some(metadata)) = database.provider(provider_name) { found = Some(metadata); break; From 6f988c681e1d2f8b237d1042d8481f5e4d19acb9 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 10 Aug 2026 11:30:09 -0400 Subject: [PATCH 59/85] fix(event-log): make saving and reordering work for macOS and keyboard operators Three findings from the thread backlog, all of them things a user hits rather than a reviewer. Saving a filter did nothing on macOS. It called window.prompt, and Tauri's macOS webview is WKWebView, which does not implement it. Replaced with an in-app field: Enter commits, Escape cancels. It also now reports when a name is refused, since save() returns null for an empty one. The column reorder arrows were unreachable by keyboard. They were Buttons nested inside a Fluent Option, which is invalid ARIA and never receives focus, because a listbox moves focus between options rather than into them. A keyboard-only operator could show and hide columns but could not order them at all. Reordering is now a separate picker with two real buttons beside it, both outside the listbox. The coverage banner mounted its live region at the same moment it gained content. A screen reader announces changes inside a region it was already tracking, so the first set of gaps was usually read as ordinary page content or not at all. The region now stays mounted and empty, and the collapse toggle reports aria-expanded. Co-Authored-By: Claude Opus 5 --- .../event-log/EvtxCoverageBanner.tsx | 11 +- src/workspaces/event-log/EvtxFilterBar.tsx | 137 +++++++++++------- 2 files changed, 96 insertions(+), 52 deletions(-) diff --git a/src/workspaces/event-log/EvtxCoverageBanner.tsx b/src/workspaces/event-log/EvtxCoverageBanner.tsx index 0045d9f49..7bdf31890 100644 --- a/src/workspaces/event-log/EvtxCoverageBanner.tsx +++ b/src/workspaces/event-log/EvtxCoverageBanner.tsx @@ -21,14 +21,20 @@ export function EvtxCoverageBanner() { const logListFontSize = useUiStore((s) => s.logListFontSize); const [collapsed, setCollapsed] = useState(false); - if (gaps.length === 0) return null; - const { fontSize, rowLineHeight } = getLogListMetrics(logListFontSize); const summary = summarizeCoverageGaps(gaps); + // The live region stays mounted and empty rather than appearing with its content. A screen + // reader announces changes inside a region it was already tracking; one that arrives already + // populated is usually read as ordinary page content, so the first gaps went unannounced. + if (gaps.length === 0) { + return
; + } + return (
setCollapsed((value) => !value)} > {collapsed ? "Show" : "Hide"} diff --git a/src/workspaces/event-log/EvtxFilterBar.tsx b/src/workspaces/event-log/EvtxFilterBar.tsx index e0202b797..4570b1956 100644 --- a/src/workspaces/event-log/EvtxFilterBar.tsx +++ b/src/workspaces/event-log/EvtxFilterBar.tsx @@ -110,13 +110,19 @@ export function EvtxFilterBar() { const resetColumns = useEvtxStore((s) => s.resetColumns); const [exportState, setExportState] = useState(null); - - const saveCurrentFilter = () => { - const name = window.prompt("Save this filter as"); - if (!name?.trim()) return; + // An in-app field rather than window.prompt. Tauri's macOS webview is WKWebView, which does not + // implement prompt, so the save-filter action silently did nothing on macOS. + const [pendingName, setPendingName] = useState(null); + const [reorderTarget, setReorderTarget] = useState(null); + const columnLabel = (id: EvtxColumnId) => + choosableColumns.find((column) => column.id === id)?.label ?? id; + + const commitFilterName = (name: string) => { + const trimmed = name.trim(); + if (!trimmed) return; const state = useEvtxStore.getState(); - saveFilter( - name, + const saved = saveFilter( + trimmed, sanitizeCriteria({ levels: [...state.filterLevels], eventIds: state.filterEventIds, @@ -125,7 +131,8 @@ export function EvtxFilterBar() { groupBy: state.groupBy, }) ); - setExportState(`Saved "${name.trim()}"`); + setPendingName(null); + setExportState(saved ? `Saved "${trimmed}"` : "That name cannot be used"); }; const applySavedFilter = (id: string) => { @@ -293,59 +300,72 @@ export function EvtxFilterBar() { value={`${columnConfig.order.length} shown`} selectedOptions={columnConfig.order} style={{ minWidth: "104px" }} - title="Choose which columns the list shows. Use the arrows to reorder." + title="Choose which columns the list shows." onOptionSelect={(_, data) => { const id = data.optionValue as EvtxColumnId | "__reset__"; if (id === "__reset__") resetColumns(); else if (id) toggleColumnVisible(id); }} > - {choosableColumns.map((column) => { - const position = columnConfig.order.indexOf(column.id); - return ( - - ); - })} + {choosableColumns.map((column) => ( + + ))} + {/* + Reordering lives outside the listbox. Buttons nested in a Fluent Option are invalid ARIA + and never receive focus, because a listbox moves focus between options rather than into + them, so a keyboard-only operator could show and hide columns but never order them. + */} + { + if (data.optionValue) setReorderTarget(data.optionValue as EvtxColumnId); + }} + > + {columnConfig.order.map((id) => ( + + ))} + + + + + { - if (data.optionValue === "__save__") saveCurrentFilter(); + if (data.optionValue === "__save__") setPendingName(""); else if (data.optionValue) applySavedFilter(data.optionValue); }} > @@ -411,6 +431,23 @@ export function EvtxFilterBar() { ))} + {pendingName !== null && ( + setPendingName(data.value)} + onKeyDown={(event) => { + if (event.key === "Enter") commitFilterName(pendingName); + if (event.key === "Escape") setPendingName(null); + }} + onBlur={() => setPendingName(null)} + /> + )} + {exportState && ( {exportState} From 3cbfe51932312450f44ff2cb7a6a15df7d8b600f Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 10 Aug 2026 11:45:48 -0400 Subject: [PATCH 60/85] fix(event-log): gate the event-log-only dependencies, and tighten weak tests Remaining findings from the thread backlog. serde_norway, rusqlite and flate2 are used only by the event-log modules but were unconditional. rusqlite carries "bundled", so a build without the feature compiled SQLite from source for code it would never call. They are optional now and pulled in by event-log, which is still part of the default set, so a normal build is unchanged. Three tests could not fail for the reason they claimed: - the malformed-XML test joined two inputs with `||`, so it passed on either and never established that an unclosed element is rejected - the NTFS lookup test searched every value for "USB", which proves nothing about the BusType binding being translated, and used "does not contain : 7" as an indirect proxy for the raw code being gone. Both now assert on the column the binding targets. - the level-fallback test asserted the length 5 rather than the fallback set, so adding a level would report a size mismatch that says nothing about the cause Also: the provider database ran a query whose HAVING clause is always true and whose LIMIT had no effect, then a second query that decided the answer anyway. One MIN/MAX query answers it. The lookup map in visibleColumns was rebuilt per call, and the row renderer calls it once per rendered row. Mapped values in the detail pane used a literal "monospace" while the rest of the file uses the shared constant. Day grouping had no test that exercised its time-zone parameter at all. Co-Authored-By: Claude Opus 5 --- .../cmtraceopen-parser/src/eventmap/path.rs | 7 +++- .../tests/eventmap_corpus.rs | 40 ++++++++++++++----- src-tauri/Cargo.toml | 17 ++++++-- src-tauri/src/event_log/event_node.rs | 11 ++++- src-tauri/src/event_log/provider_db.rs | 20 ++++------ src-tauri/src/state/app_state.rs | 2 +- src/workspaces/event-log/EvtxDetailPane.tsx | 2 +- src/workspaces/event-log/evtx-columns.ts | 19 ++++++--- .../event-log/evtx-grouping.test.ts | 33 +++++++++++++++ .../event-log/evtx-saved-filters.test.ts | 5 ++- .../event-log/evtx-saved-filters.ts | 3 +- 11 files changed, 119 insertions(+), 40 deletions(-) diff --git a/crates/cmtraceopen-parser/src/eventmap/path.rs b/crates/cmtraceopen-parser/src/eventmap/path.rs index a07ba86cf..cf20c95e9 100644 --- a/crates/cmtraceopen-parser/src/eventmap/path.rs +++ b/crates/cmtraceopen-parser/src/eventmap/path.rs @@ -37,7 +37,12 @@ pub enum PathError { Empty(String), /// A predicate was opened but not closed, or was not understood. #[error("value path has a malformed predicate in step '{step}': {path}")] - MalformedPredicate { path: String, step: String }, + MalformedPredicate { + /// The whole expression, so the map entry it came from can be identified. + path: String, + /// The single step that would not parse, which is the part to correct. + step: String, + }, /// An attribute selector appeared somewhere other than the final step. #[error("value path may only select an attribute in its final step: {0}")] MisplacedAttribute(String), diff --git a/crates/cmtraceopen-parser/tests/eventmap_corpus.rs b/crates/cmtraceopen-parser/tests/eventmap_corpus.rs index 4189841bd..c4f400c25 100644 --- a/crates/cmtraceopen-parser/tests/eventmap_corpus.rs +++ b/crates/cmtraceopen-parser/tests/eventmap_corpus.rs @@ -137,28 +137,48 @@ fn ntfs_146_applies_its_lookup_table_and_default() { let lookup = map.lookup_for("BusType").expect("BusType lookup present"); assert_eq!(lookup.default.as_deref(), Some("Unknown code")); + // Asserted on the column the binding targets, not on "some column somewhere". `any` over every + // value passed if the string turned up anywhere, so it did not establish that the BusType + // binding itself was translated, and "does not contain : 7" was only an indirect proxy for + // the raw code having been replaced. let usb = apply_map( &map, &event_with_named_data(&[("VolumeName", "C:"), ("BusType", "7")]), ); + let translated = usb + .values + .iter() + .find(|value| value.text.contains("USB")) + .unwrap_or_else(|| { + panic!( + "no value rendered the BusType translation: {:?}", + usb.values.iter().map(|v| &v.text).collect::>() + ) + }); assert!( - usb.values - .iter() - .any(|value| value.text.contains("USB") && !value.text.contains(": 7")), - "raw BusType 7 should render as USB: {:?}", - usb.values.iter().map(|v| &v.text).collect::>() + !translated.text.contains('7'), + "the raw code should be gone once translated: {}", + translated.text ); let unknown = apply_map( &map, &event_with_named_data(&[("VolumeName", "C:"), ("BusType", "255")]), ); + let defaulted = unknown + .values + .iter() + .find(|value| value.text.contains("Unknown code")) + .unwrap_or_else(|| { + panic!( + "an out-of-table code should fall back to the lookup default: {:?}", + unknown.values.iter().map(|v| &v.text).collect::>() + ) + }); assert!( - unknown - .values - .iter() - .any(|value| value.text.contains("Unknown code")), - "an out-of-table code should fall back to the lookup default" + !defaulted.text.contains("255"), + "the default replaces the code rather than appending to it: {}", + defaulted.text ); } diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 611c5575f..ecd4e045d 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -25,7 +25,13 @@ tauri-build = { version = "2", features = [] } default = ["full"] full = ["collector", "deployment", "dsregcmd", "esp-diagnostics", "event-log", "intune-diagnostics", "macos-diag", "sccm-diagnostics", "secureboot", "sysmon"] sysmon = ["dep:evtx"] -event-log = ["dep:evtx", "dep:quick-xml"] +event-log = [ + "dep:evtx", + "dep:quick-xml", + "dep:serde_norway", + "dep:rusqlite", + "dep:flate2", +] collector = [] deployment = [] dsregcmd = ["intune-diagnostics"] @@ -75,9 +81,12 @@ cab = { version = "=0.6.0", optional = true } # using only std::time -- so the whole graph (cab, cookie, plist, simplelog, # tauri-plugin-log, tauri-plugin-updater) is held at or above the patched release. time = { version = "0.3.47", default-features = false, features = ["parsing"] } -serde_norway = "0.9.42" -rusqlite = { version = "0.40.2", features = ["bundled"] } -flate2 = "1.1.9" +# Only the event-log modules use these, so they are optional and pulled in by that feature. +# rusqlite carries "bundled", which compiles SQLite from source; an unrelated build should not pay +# for it. +serde_norway = { version = "0.9.42", optional = true } +rusqlite = { version = "0.40.2", features = ["bundled"], optional = true } +flate2 = { version = "1.1.9", optional = true } [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/src-tauri/src/event_log/event_node.rs b/src-tauri/src/event_log/event_node.rs index b765218ba..72173d26a 100644 --- a/src-tauri/src/event_log/event_node.rs +++ b/src-tauri/src/event_log/event_node.rs @@ -381,8 +381,15 @@ mod tests { #[test] fn malformed_xml_is_an_error_rather_than_a_partial_tree() { - assert!(parse_event_xml("").is_err() || parse_event_xml("").is_err(), + "an unclosed element must not yield a partial tree" + ); + assert!(parse_event_xml(" = connection .query_row( - "SELECT SourceOsBuild FROM ProviderDetails \ - GROUP BY SourceOsBuild HAVING COUNT(DISTINCT SourceOsBuild) >= 0 LIMIT 2", + "SELECT MIN(SourceOsBuild), MAX(SourceOsBuild) FROM ProviderDetails", [], - |row| row.get(0), + |row| Ok((row.get::<_, Option>(0)?, row.get::<_, Option>(1)?)), ) .ok() - .filter(|_| { - connection - .query_row( - "SELECT COUNT(DISTINCT SourceOsBuild) FROM ProviderDetails", - [], - |row| row.get::<_, u32>(0), - ) - .map(|distinct| distinct == 1) - .unwrap_or(false) + .and_then(|(low, high)| match (low, high) { + (Some(low), Some(high)) if low == high => Some(low), + _ => None, }); Ok(Self { diff --git a/src-tauri/src/state/app_state.rs b/src-tauri/src/state/app_state.rs index ad7a6caa9..50d3d1265 100644 --- a/src-tauri/src/state/app_state.rs +++ b/src-tauri/src/state/app_state.rs @@ -86,7 +86,7 @@ impl AppState { #[cfg(feature = "esp-diagnostics")] esp_session_manager: Mutex::new(None), #[cfg(feature = "event-log")] - event_maps: Arc::new(RwLock::new(Default::default())), + event_maps: Arc::new(RwLock::new(cmtraceopen_parser::eventmap::MapRegistry::new())), #[cfg(feature = "event-log")] provider_store: Arc::new(RwLock::new(ProviderStore::default())), } diff --git a/src/workspaces/event-log/EvtxDetailPane.tsx b/src/workspaces/event-log/EvtxDetailPane.tsx index 415699ba4..e6dd33121 100644 --- a/src/workspaces/event-log/EvtxDetailPane.tsx +++ b/src/workspaces/event-log/EvtxDetailPane.tsx @@ -253,7 +253,7 @@ export function EvtxDetailPane() { display: "flex", gap: "8px", fontSize: `${monoFontSize}px`, - fontFamily: "monospace", + fontFamily: LOG_MONOSPACE_FONT_FAMILY, }} > (EVTX_COLUMNS.map((column) => column.id)); +/** + * Fixed columns keyed by id. + * + * Module scope because EVTX_COLUMNS never changes and the row renderer calls visibleColumns once + * per rendered row; rebuilding the map there allocated one per row per render. + */ +const FIXED_COLUMNS_BY_ID = new Map( + EVTX_COLUMNS.map((column) => [column.id, column]) +); + /** * Whether a stored id is one this build can render. * @@ -129,7 +139,7 @@ const COLUMN_IDS = new Set(EVTX_COLUMNS.map((column) => column.id)); */ function isKnownColumnId(candidate: string): boolean { if (COLUMN_IDS.has(candidate)) return true; - const property = mappedColumnProperty(candidate as EvtxColumnId); + const property = mappedColumnProperty(candidate); return property !== null && property.length > 0; } @@ -190,12 +200,9 @@ export function sanitizeColumnConfig(input: unknown): EvtxColumnConfig { /** Specs for the visible columns, in display order. */ export function visibleColumns(config: EvtxColumnConfig): EvtxColumnSpec[] { - const byId = new Map( - EVTX_COLUMNS.map((column) => [column.id, column]) - ); return config.order .map((id) => { - const fixed = byId.get(id); + const fixed = FIXED_COLUMNS_BY_ID.get(id); if (fixed) return fixed; // Synthesized from the id, so rendering a map column needs no knowledge of which maps are // loaded. That keeps the row renderer independent of load order. diff --git a/src/workspaces/event-log/evtx-grouping.test.ts b/src/workspaces/event-log/evtx-grouping.test.ts index 57476cc64..1c7ae9ed4 100644 --- a/src/workspaces/event-log/evtx-grouping.test.ts +++ b/src/workspaces/event-log/evtx-grouping.test.ts @@ -204,3 +204,36 @@ describe("group key encoding", () => { expect(visible[0].record.id).toBe(1); }); }); + +describe("day grouping honours the time zone", () => { + it("buckets by UTC date when asked for UTC", () => { + // Every other test here relies on the default, so a regression that ignored the argument and + // always bucketed by local date would pass the whole file. This instant sits on either side of + // midnight depending on the zone. + const lateUtc = Date.UTC(2026, 1, 10, 23, 30, 0); + const rows = buildGroupedRows( + [record({ id: 1, timestampEpoch: lateUtc })], + ["day"], + new Set(), + "utc" + ); + const header = rows.find((r) => r.kind === "group"); + expect(header && header.kind === "group" ? header.label : "").toBe("2026-02-10"); + }); + + it("buckets by the machine's own date when asked for local", () => { + const lateUtc = Date.UTC(2026, 1, 10, 23, 30, 0); + const local = new Date(lateUtc); + const expected = + `${local.getFullYear()}-${String(local.getMonth() + 1).padStart(2, "0")}-` + + `${String(local.getDate()).padStart(2, "0")}`; + const rows = buildGroupedRows( + [record({ id: 1, timestampEpoch: lateUtc })], + ["day"], + new Set(), + "local" + ); + const header = rows.find((r) => r.kind === "group"); + expect(header && header.kind === "group" ? header.label : "").toBe(expected); + }); +}); diff --git a/src/workspaces/event-log/evtx-saved-filters.test.ts b/src/workspaces/event-log/evtx-saved-filters.test.ts index fcdb68da7..f6f84cbb8 100644 --- a/src/workspaces/event-log/evtx-saved-filters.test.ts +++ b/src/workspaces/event-log/evtx-saved-filters.test.ts @@ -5,6 +5,7 @@ import { orderFilters, parseFilterExport, SAVED_FILTER_SCHEMA, + ALL_LEVELS, sanitizeCriteria, sanitizeSavedFilter, type EvtxSavedFilter, @@ -48,7 +49,9 @@ describe("sanitizeCriteria", () => { it("falls back to every level when none survive, rather than matching nothing", () => { const criteria = sanitizeCriteria({ levels: ["Bogus"] }); - expect(criteria.levels).toHaveLength(5); + // Compared against the constant rather than its current size, so adding a level does not + // report a length mismatch that says nothing about the cause. + expect([...criteria.levels].sort()).toEqual([...ALL_LEVELS].sort()); }); it("falls back to a known time window", () => { diff --git a/src/workspaces/event-log/evtx-saved-filters.ts b/src/workspaces/event-log/evtx-saved-filters.ts index 26be3e144..1578fd835 100644 --- a/src/workspaces/event-log/evtx-saved-filters.ts +++ b/src/workspaces/event-log/evtx-saved-filters.ts @@ -27,7 +27,8 @@ export interface EvtxSavedFilter { lastUsed: number | null; } -const ALL_LEVELS: EvtxLevel[] = [ +/** Every level, and the fallback when a stored filter names none this build recognizes. */ +export const ALL_LEVELS: EvtxLevel[] = [ "Critical", "Error", "Warning", From 2965fbf76ef817534a21b348cadb6e4456ac7520 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 10 Aug 2026 12:11:19 -0400 Subject: [PATCH 61/85] fix(parser): make provider metadata able to read its own output back The public provider types deserialized .NET's signed integers and serialized them unsigned, so the round trip was asymmetric: the reserved keyword 0x8000000000000000 arrives as -9223372036854775808, was written back as 9223372036854775808, and then failed to deserialize because the reader expects i64. Anything persisting or forwarding this metadata could not re-read what it had just written. Matching serializers now preserve the signed form the source used, and a test asserts the value survives a full cycle. Also from the thread backlog: - delimited export allocated a Vec and a fresh separator String per row, on an export that can run to a hundred thousand records. Fields are written straight into the output. - nothing compared the record's JSON wire names against the TypeScript that reads them, so a rename or a missed serde attribute would have surfaced only as undefined in the detail pane. A test pins the camelCase keys and asserts no snake_case leaks. Co-Authored-By: Claude Opus 5 --- crates/cmtraceopen-parser/src/provider/mod.rs | 72 +++++++++++++++- src-tauri/src/event_log/export.rs | 83 +++++++++++++++---- 2 files changed, 136 insertions(+), 19 deletions(-) diff --git a/crates/cmtraceopen-parser/src/provider/mod.rs b/crates/cmtraceopen-parser/src/provider/mod.rs index 5ce5e3a86..d74058aa3 100644 --- a/crates/cmtraceopen-parser/src/provider/mod.rs +++ b/crates/cmtraceopen-parser/src/provider/mod.rs @@ -15,7 +15,8 @@ use std::collections::BTreeMap; -use serde::{Deserialize, Deserializer, Serialize}; +use serde::ser::SerializeSeq; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; /// Reinterprets a signed integer as unsigned, preserving the bit pattern. /// @@ -35,6 +36,28 @@ fn signed_as_u64_vec<'de, D: Deserializer<'de>>(deserializer: D) -> Result(value: &u64, serializer: S) -> Result { + serializer.serialize_i64(*value as i64) +} + +fn u64_vec_as_signed(values: &[u64], serializer: S) -> Result { + let mut sequence = serializer.serialize_seq(Some(values.len()))?; + for value in values { + sequence.serialize_element(&(*value as i64))?; + } + sequence.end() +} + +fn u32_as_signed(value: &u32, serializer: S) -> Result { + serializer.serialize_i64(*value as i32 as i64) +} + /// Reinterprets a signed integer as an unsigned 32-bit value. /// /// Message identifiers above `0x7FFFFFFF` are written negative for the same reason. @@ -67,7 +90,11 @@ pub struct ProviderEvent { #[serde(default)] pub opcode: Option, /// Keyword bitmask values. - #[serde(default, deserialize_with = "signed_as_u64_vec")] + #[serde( + default, + deserialize_with = "signed_as_u64_vec", + serialize_with = "u64_vec_as_signed" + )] pub keywords: Vec, /// The manifest template, which declares each field's name and type. #[serde(default)] @@ -79,10 +106,18 @@ pub struct ProviderEvent { #[serde(rename_all = "PascalCase")] pub struct ProviderMessage { /// Full message identifier as the provider declares it. - #[serde(default, deserialize_with = "signed_as_u64")] + #[serde( + default, + deserialize_with = "signed_as_u64", + serialize_with = "u64_as_signed" + )] pub raw_id: u64, /// Low bits of `raw_id`, which is what most references use. - #[serde(default, deserialize_with = "signed_as_u32")] + #[serde( + default, + deserialize_with = "signed_as_u32", + serialize_with = "u32_as_signed" + )] pub short_id: u32, /// The message text. #[serde(default)] @@ -494,4 +529,33 @@ mod tests { assert_eq!(metadata.keyword_names(0b10_0001), vec!["Startup", "Memory"]); assert!(metadata.keyword_names(0).is_empty()); } + + #[test] + fn a_top_bit_keyword_survives_a_serialization_round_trip() { + // The .NET source writes these signed. Deserializing accepted that and serializing wrote + // the unsigned form, so the type could not read its own output back: anything that + // persisted or forwarded provider metadata broke on the reserved keyword alone. + let json = r#"{"Id":1,"Version":0,"Keywords":[-9223372036854775808]}"#; + let event: ProviderEvent = serde_json::from_str(json).expect("deserializes"); + assert_eq!(event.keywords, vec![0x8000_0000_0000_0000]); + + let written = serde_json::to_string(&event).expect("serializes"); + assert!( + written.contains("-9223372036854775808"), + "the signed form the source used must be preserved: {written}" + ); + + let again: ProviderEvent = serde_json::from_str(&written).expect("re-reads its own output"); + assert_eq!(again, event); + } + + #[test] + fn an_ordinary_keyword_is_unchanged_by_the_round_trip() { + let json = r#"{"Id":1,"Version":0,"Keywords":[16]}"#; + let event: ProviderEvent = serde_json::from_str(json).expect("deserializes"); + let written = serde_json::to_string(&event).expect("serializes"); + assert!(written.contains("[16]"), "{written}"); + let again: ProviderEvent = serde_json::from_str(&written).expect("re-reads"); + assert_eq!(again.keywords, vec![16]); + } } diff --git a/src-tauri/src/event_log/export.rs b/src-tauri/src/event_log/export.rs index de1cfcb71..3a0ae57e8 100644 --- a/src-tauri/src/event_log/export.rs +++ b/src-tauri/src/event_log/export.rs @@ -173,26 +173,33 @@ pub fn export_records(records: &[EvtxRecord], format: ExportFormat) -> Result { let delimiter = format.delimiter(); let mapped = mapped_columns(records); + + // Written straight into the output. Collecting each row into a Vec and joining it + // allocated a vector and a fresh separator String per record, on an export that can + // run to a hundred thousand of them. let mut out = String::new(); - out.push_str( - &COLUMNS + let write_row = |out: &mut String, cells: &mut dyn Iterator| { + let mut first = true; + for cell in cells { + if !first { + out.push(delimiter); + } + first = false; + out.push_str(&escape_delimited(cell, delimiter)); + } + out.push('\n'); + }; + + write_row( + &mut out, + &mut COLUMNS .iter() - .map(|column| column.to_string()) - .chain(mapped.iter().cloned()) - .map(|column| escape_delimited(&column, delimiter)) - .collect::>() - .join(&delimiter.to_string()), + .copied() + .chain(mapped.iter().map(String::as_str)), ); - out.push('\n'); for record in records { let row = row_of(record, &mapped); - out.push_str( - &row.iter() - .map(|value| escape_delimited(value, delimiter)) - .collect::>() - .join(&delimiter.to_string()), - ); - out.push('\n'); + write_row(&mut out, &mut row.iter().map(String::as_str)); } Ok(out) } @@ -376,6 +383,52 @@ mod tests { assert!(out.contains("\"a\tb\"")); } + #[test] + fn json_carries_the_metadata_fields_with_the_wire_names_the_frontend_reads() { + // The TypeScript EvtxRecord declares these in camelCase. Nothing on either side compares + // the two, so a rename or a missed serde attribute would surface only as undefined in the + // detail pane. + let mut r = record("x"); + r.mapped = vec![crate::event_log::maps::MappedColumn { + property: "PayloadData1".into(), + text: "cmd.exe".into(), + complete: true, + }]; + let json: serde_json::Value = + serde_json::from_str(&export_records(&[r], ExportFormat::Json).expect("exports")) + .expect("valid JSON"); + let first = &json[0]; + + // Presence, not value: the fixture leaves some of these unset on purpose, and an absent + // optional legitimately serializes as null. What matters is that the key is there and + // spelled the way the frontend reads it. + for key in [ + "eventRecordId", + "timestampEpoch", + "sourceLabel", + "processId", + "threadId", + "userSid", + "eventData", + "rawXml", + "mapped", + ] { + assert!(first.get(key).is_some(), "{key} missing: {first}"); + } + assert_eq!(first["eventRecordId"], 42); + assert_eq!(first["processId"], 1234); + assert_eq!(first["userSid"], "S-1-5-18"); + assert_eq!(first["mapped"][0]["property"], "PayloadData1"); + for snake in [ + "event_record_id", + "timestamp_epoch", + "source_label", + "user_sid", + ] { + assert!(first.get(snake).is_none(), "{snake} leaked in snake_case"); + } + } + #[test] fn json_round_trips_back_into_records() { let out = export_records(&[record("hello")], ExportFormat::Json).expect("exports"); From 270b557b64e654dca5b4bccfa6df2c5d599747a8 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 10 Aug 2026 12:18:32 -0400 Subject: [PATCH 62/85] perf(event-log): stop shipping every record's XML for a delimited export The export command receives records over IPC and raw_xml dominates that payload, but only the XML and JSON formats read it. A CSV export of a hundred thousand records serialized every record's XML across the bridge for nothing. The frontend now omits it for CSV and TSV, and raw_xml and event_data default so a trimmed payload deserializes. Tests assert the delimited output is identical with and without it. Co-Authored-By: Claude Opus 5 --- src-tauri/src/event_log/export.rs | 25 ++++++++++++++++++++++ src-tauri/src/event_log/models.rs | 7 ++++++ src/workspaces/event-log/EvtxFilterBar.tsx | 9 +++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src-tauri/src/event_log/export.rs b/src-tauri/src/event_log/export.rs index 3a0ae57e8..401bc4d11 100644 --- a/src-tauri/src/event_log/export.rs +++ b/src-tauri/src/event_log/export.rs @@ -480,6 +480,31 @@ mod tests { assert!(xml.contains("") && xml.contains("")); } + #[test] + fn a_delimited_export_works_from_a_payload_without_raw_xml() { + // The frontend omits rawXml for CSV and TSV because neither reads it and it dominates the + // IPC payload. Deserializing has to tolerate that, and the output must be unchanged. + let mut trimmed = record("x"); + trimmed.raw_xml = String::new(); + + let with_xml = export_records(&[record("x")], ExportFormat::Csv).expect("exports"); + let without = export_records(&[trimmed], ExportFormat::Csv).expect("exports"); + assert_eq!(with_xml, without); + } + + #[test] + fn a_record_missing_raw_xml_still_deserializes() { + let json = serde_json::json!([{ + "id": 0, "eventRecordId": 1, "timestamp": "", "timestampEpoch": 0, + "provider": "P", "channel": "C", "eventId": 1, "level": "Error", + "computer": "TESTHOST-01", "message": "m", "sourceLabel": "Live", "mapped": [] + }]); + let records: Vec = + serde_json::from_value(json).expect("a trimmed payload deserializes"); + assert_eq!(records[0].raw_xml, ""); + assert!(records[0].event_data.is_empty()); + } + #[test] fn extensions_match_the_format() { assert_eq!(ExportFormat::Csv.extension(), "csv"); diff --git a/src-tauri/src/event_log/models.rs b/src-tauri/src/event_log/models.rs index f8f896aa0..911f67ba9 100644 --- a/src-tauri/src/event_log/models.rs +++ b/src-tauri/src/event_log/models.rs @@ -13,7 +13,14 @@ pub struct EvtxRecord { pub level: EvtxLevel, pub computer: String, pub message: String, + #[serde(default)] pub event_data: Vec, + /// The provider's own XML. + /// + /// Defaulted so a caller can omit it. The export command receives records over IPC, and this + /// field dominates the payload: only the XML and JSON formats read it, so sending it for a + /// delimited export serialized every record's XML across the bridge for nothing. + #[serde(default)] pub raw_xml: String, pub source_label: String, /// Provider-defined task grouping, when the event declares one. diff --git a/src/workspaces/event-log/EvtxFilterBar.tsx b/src/workspaces/event-log/EvtxFilterBar.tsx index 4570b1956..b1df6cc89 100644 --- a/src/workspaces/event-log/EvtxFilterBar.tsx +++ b/src/workspaces/event-log/EvtxFilterBar.tsx @@ -166,8 +166,15 @@ export function EvtxFilterBar() { }); if (!destination) return; setExportState("Exporting..."); + // Only the XML and JSON formats read rawXml, and it dominates the payload: sending it for a + // delimited export serializes every record's XML across the IPC bridge for nothing, which on + // a hundred-thousand-record export is the bulk of the transfer. + const payload = + format.value === "csv" || format.value === "tsv" + ? records.map(({ rawXml: _rawXml, ...rest }) => rest) + : records; const bytes = await invoke("evtx_export_records", { - records, + records: payload, format: format.value, destination, }); From 91273d824311801a3b4518934e6385c439b02072 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 10 Aug 2026 13:12:57 -0400 Subject: [PATCH 63/85] fix(event-log): verify the reader's reply at the boundary, once Reconsidered a review finding I had declined. I objected to per-handler validation of IPC responses, and I still do: the producer is this repository's own typed Rust and five bespoke checks would set a precedent the rest of the IPC surface does not follow. The cheap version of the argument holds though. The store destructures and iterates three fields, and if a future backend change dropped errorMessages, spreading it would throw somewhere unrelated and surface as a confusing load error rather than as a contract mismatch. One shared guard at the boundary names the contract instead, and costs five lines at the call sites. It throws rather than returning a default, because a reply the store cannot read is not a reply with no events, and quietly showing an empty list is the failure this workspace exists to avoid. An absent errorMessages is still treated as no gaps, since that is an older reader reporting nothing rather than a malformed reply. Co-Authored-By: Claude Opus 5 --- .../event-log/evtx-coverage.test.ts | 40 ++++++++++++++++++- src/workspaces/event-log/evtx-coverage.ts | 30 ++++++++++++++ src/workspaces/event-log/evtx-store.ts | 6 ++- 3 files changed, 74 insertions(+), 2 deletions(-) diff --git a/src/workspaces/event-log/evtx-coverage.test.ts b/src/workspaces/event-log/evtx-coverage.test.ts index 5405a4f6f..4db654f86 100644 --- a/src/workspaces/event-log/evtx-coverage.test.ts +++ b/src/workspaces/event-log/evtx-coverage.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "vitest"; -import { mergeCoverageGaps, summarizeCoverageGaps } from "./evtx-coverage"; +import { + assertParseResultShape, + mergeCoverageGaps, + summarizeCoverageGaps, +} from "./evtx-coverage"; describe("mergeCoverageGaps", () => { it("accumulates gaps across channels", () => { @@ -59,3 +63,37 @@ describe("gaps across load paths", () => { expect(second).toHaveLength(2); }); }); + +describe("assertParseResultShape", () => { + it("accepts a well formed reply", () => { + const shape = assertParseResultShape({ + records: [], + channels: [], + errorMessages: ["Application: 3 records unreadable"], + }); + expect(shape.errorMessages).toEqual(["Application: 3 records unreadable"]); + }); + + it("treats absent errorMessages as no gaps rather than a malformed reply", () => { + // An older reader that reports nothing is not the same as one this build cannot read. + expect(assertParseResultShape({ records: [], channels: [] }).errorMessages).toEqual([]); + }); + + it("rejects a reply whose records are not a list", () => { + // Spreading this would throw somewhere unrelated and surface as a confusing load error. + expect(() => assertParseResultShape({ records: null, channels: [] })).toThrow( + /cannot read/ + ); + expect(() => assertParseResultShape({ channels: [] })).toThrow(/cannot read/); + expect(() => assertParseResultShape(undefined)).toThrow(/cannot read/); + }); + + it("drops non-string gap entries rather than rendering them", () => { + const shape = assertParseResultShape({ + records: [], + channels: [], + errorMessages: ["real", 42, null], + }); + expect(shape.errorMessages).toEqual(["real"]); + }); +}); diff --git a/src/workspaces/event-log/evtx-coverage.ts b/src/workspaces/event-log/evtx-coverage.ts index 3b3eb3cda..0d68bf93a 100644 --- a/src/workspaces/event-log/evtx-coverage.ts +++ b/src/workspaces/event-log/evtx-coverage.ts @@ -27,3 +27,33 @@ export function mergeCoverageGaps( export function summarizeCoverageGaps(gaps: readonly string[]): string { return gaps.length === 1 ? "1 gap in this view" : `${gaps.length} gaps in this view`; } + +/** + * The parts of an event-log IPC reply the store reads, verified once. + * + * Not a schema validator, and deliberately not per-handler checks: it guards the three fields the + * store destructures and iterates. If a future backend change dropped `errorMessages`, spreading + * it would throw somewhere unrelated and surface as a confusing load error; this fails at the + * boundary with a message that names the contract. + * + * Throws rather than returning a default, because a reply the store cannot read is not a reply + * with no events, and quietly showing an empty list is the failure this workspace exists to avoid. + */ +export function assertParseResultShape(value: unknown): { + records: unknown[]; + channels: unknown[]; + errorMessages: string[]; +} { + const reply = value as { records?: unknown; channels?: unknown; errorMessages?: unknown }; + if (!Array.isArray(reply?.records) || !Array.isArray(reply?.channels)) { + throw new Error("the event log reader returned a reply this build cannot read"); + } + return { + records: reply.records, + channels: reply.channels, + // Absent means the reader reported no gaps, which is different from a malformed reply. + errorMessages: Array.isArray(reply.errorMessages) + ? reply.errorMessages.filter((entry): entry is string => typeof entry === "string") + : [], + }; +} diff --git a/src/workspaces/event-log/evtx-store.ts b/src/workspaces/event-log/evtx-store.ts index 291d8200e..8f827a84e 100644 --- a/src/workspaces/event-log/evtx-store.ts +++ b/src/workspaces/event-log/evtx-store.ts @@ -1,5 +1,5 @@ import { create } from "zustand"; -import { mergeCoverageGaps } from "./evtx-coverage"; +import { assertParseResultShape, mergeCoverageGaps } from "./evtx-coverage"; import type { EvtxTimeZoneMode } from "./evtx-time"; import { invoke } from "@tauri-apps/api/core"; import { listen } from "@tauri-apps/api/event"; @@ -153,6 +153,7 @@ export const useEvtxStore = create()((set, get) => ({ set({ isLoading: true, loadError: null }); try { const result = await invoke("evtx_parse_files", { paths }); + assertParseResultShape(result); set(applyParseResult(result, "files")); } catch (error) { const message = error instanceof Error ? error.message : String(error); @@ -229,6 +230,7 @@ export const useEvtxStore = create()((set, get) => ({ maxEvents: null, filter: buildServerFilter(get().timeWindow), }); + assertParseResultShape(result); mergeResult(ch, result); } catch (e) { const msg = e instanceof Error ? e.message : String(e); @@ -260,6 +262,7 @@ export const useEvtxStore = create()((set, get) => ({ maxEvents: maxEvents ?? null, filter: buildServerFilter(get().timeWindow), }); + assertParseResultShape(result); // Merge new records with existing ones (for incremental channel loading) const state = get(); @@ -335,6 +338,7 @@ export const useEvtxStore = create()((set, get) => ({ // outside the window the toolbar was still showing as selected. filter: buildServerFilter(get().timeWindow), }); + assertParseResultShape(result); const s = get(); const merged = [...s.records, ...result.records]; From ba55802b76640940047addefb313e033148baea7 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 10 Aug 2026 14:05:15 -0400 Subject: [PATCH 64/85] fix(event-log): keep the empty live region in the accessibility tree Three follow-ups, one of them a mistake in the previous fix. I made the empty coverage region `display: "none"`, which removes it from the accessibility tree entirely and so defeats the point of mounting it early: the first gaps still arrived in a newly exposed region and went unannounced. The region is now always rendered and unstyled when empty, with the banner content conditional inside it. The delimited export payload also carried eventData, which the delimited writers do not read, so that is omitted alongside rawXml. The export byte count is checked with Number.isSafeInteger rather than Number.isFinite. A count past MAX_SAFE_INTEGER is finite and would format into a confidently wrong size. Co-Authored-By: Claude Opus 5 --- .../event-log/EvtxCoverageBanner.tsx | 46 +++++++++++-------- src/workspaces/event-log/EvtxFilterBar.tsx | 12 +++-- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/src/workspaces/event-log/EvtxCoverageBanner.tsx b/src/workspaces/event-log/EvtxCoverageBanner.tsx index 7bdf31890..c390e1f83 100644 --- a/src/workspaces/event-log/EvtxCoverageBanner.tsx +++ b/src/workspaces/event-log/EvtxCoverageBanner.tsx @@ -24,31 +24,37 @@ export function EvtxCoverageBanner() { const { fontSize, rowLineHeight } = getLogListMetrics(logListFontSize); const summary = summarizeCoverageGaps(gaps); - // The live region stays mounted and empty rather than appearing with its content. A screen - // reader announces changes inside a region it was already tracking; one that arrives already - // populated is usually read as ordinary page content, so the first gaps went unannounced. - if (gaps.length === 0) { - return
; - } + // The live region is always rendered, and the banner content appears inside it. A screen reader + // announces changes within a region it was already tracking, so a region that arrives already + // populated is read as ordinary page content and the first gaps go unannounced. It must also + // stay in the accessibility tree while empty, which display:none would prevent, so an empty + // region is simply an unstyled element with no children. + const empty = gaps.length === 0; return (
+ {empty ? null : ( + <>
@@ -71,6 +77,8 @@ export function EvtxCoverageBanner() { ))} )} + + )}
); } diff --git a/src/workspaces/event-log/EvtxFilterBar.tsx b/src/workspaces/event-log/EvtxFilterBar.tsx index b1df6cc89..bb3076e7a 100644 --- a/src/workspaces/event-log/EvtxFilterBar.tsx +++ b/src/workspaces/event-log/EvtxFilterBar.tsx @@ -166,12 +166,12 @@ export function EvtxFilterBar() { }); if (!destination) return; setExportState("Exporting..."); - // Only the XML and JSON formats read rawXml, and it dominates the payload: sending it for a - // delimited export serializes every record's XML across the IPC bridge for nothing, which on - // a hundred-thousand-record export is the bulk of the transfer. + // The delimited writers read neither rawXml nor eventData, and rawXml dominates the payload: + // sending them for a CSV export serializes every record's XML and field list across the IPC + // bridge for nothing, which on a hundred-thousand-record export is the bulk of the transfer. const payload = format.value === "csv" || format.value === "tsv" - ? records.map(({ rawXml: _rawXml, ...rest }) => rest) + ? records.map(({ rawXml: _rawXml, eventData: _eventData, ...rest }) => rest) : records; const bytes = await invoke("evtx_export_records", { records: payload, @@ -181,7 +181,9 @@ export function EvtxFilterBar() { // The IPC boundary is typed by assertion, not by the compiler. A malformed reply would // otherwise render as "Exported ... (NaN KB)", which still reads as success, and an operator // would believe a file was written. - if (typeof bytes !== "number" || !Number.isFinite(bytes) || bytes < 0) { + // A safe integer, not merely finite: a count past Number.MAX_SAFE_INTEGER would format into + // a confidently wrong size. + if (typeof bytes !== "number" || !Number.isSafeInteger(bytes) || bytes < 0) { setExportState("Export failed: the writer did not report how much it wrote"); return; } From 20c01bee9d4c4493c7da5024f8d93eb8e4f8b7b1 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 10 Aug 2026 15:13:18 -0400 Subject: [PATCH 65/85] fix(event-log): refuse malformed value paths, and declare the grouped list a tree The IPC guard I added last round normalized its input and then every caller ignored the return value and used the raw reply, so a gap list of ["real gap", 42] still stored 42. The guard is only worth having if what it returns is what gets used; it is now, with a test that would have caught it. The path parser filtered empty segments away, so /Event//EventData/Data and /Event/@ parsed as though they were the path the author meant. A typo in a map should be reported, not silently reinterpreted into a different path. A single trailing slash stays benign, since /Event/EventData/ names the same element. The scroll container declared role="listbox" even when grouped, and a listbox may only own option and group children, so the group headers were invalid and assistive technology could drop them or stop treating the rows as a set. It is a tree once grouped, with the headers as treeitems carrying their level. Also: the column specs were rebuilt per rendered row although columnConfig is stable, so a hundred thousand rows re-synthesized every map column spec; the reset entry sat in a multiselect listbox where every option renders a selection indicator, so an action looked like a column that could be checked, and it is a button now; and registering provider databases published info before the locks were taken, so a poisoned lock left registered() reporting coverage no lookup could deliver. Co-Authored-By: Claude Opus 5 --- .../cmtraceopen-parser/src/eventmap/path.rs | 75 ++++++++++++++++++- src-tauri/src/event_log/provider_db.rs | 22 ++++-- src/workspaces/event-log/EvtxFilterBar.tsx | 23 ++++-- src/workspaces/event-log/EvtxTimeline.tsx | 15 +++- src/workspaces/event-log/EvtxTimelineRow.tsx | 7 +- .../event-log/evtx-store-coverage.test.ts | 19 +++++ src/workspaces/event-log/evtx-store.ts | 23 +++--- 7 files changed, 153 insertions(+), 31 deletions(-) diff --git a/crates/cmtraceopen-parser/src/eventmap/path.rs b/crates/cmtraceopen-parser/src/eventmap/path.rs index cf20c95e9..fe10afeb0 100644 --- a/crates/cmtraceopen-parser/src/eventmap/path.rs +++ b/crates/cmtraceopen-parser/src/eventmap/path.rs @@ -43,6 +43,17 @@ pub enum PathError { /// The single step that would not parse, which is the part to correct. step: String, }, + /// A step was empty, from a doubled or dangling separator, or an attribute with no name. + /// + /// Refused rather than skipped: `/Event//EventData/Data` is a typo, and quietly reading it as + /// `/Event/EventData/Data` would apply a map the author did not write. + #[error("value path has an empty step '{step}': {path}")] + MalformedStep { + /// The whole expression, so the map entry it came from can be identified. + path: String, + /// The offending step, empty for a doubled separator. + step: String, + }, /// An attribute selector appeared somewhere other than the final step. #[error("value path may only select an attribute in its final step: {0}")] MisplacedAttribute(String), @@ -52,7 +63,12 @@ pub enum PathError { #[derive(Debug, Clone, PartialEq, Eq)] pub enum Predicate { /// `[@Name="value"]`, selecting the first sibling whose attribute matches. - AttributeEquals { name: String, value: String }, + AttributeEquals { + /// Attribute to compare, without the leading `@`. + name: String, + /// Value it must equal, already unquoted. + value: String, + }, /// `[n]`, a 1-based index across same-named siblings, as XPath numbers them. Index(usize), } @@ -81,15 +97,39 @@ impl ValuePath { return Err(PathError::NotAbsolute(expression.to_string())); }; + if body.trim().is_empty() { + return Err(PathError::Empty(expression.to_string())); + } + let mut steps = Vec::new(); let mut attribute = None; - let segments: Vec<&str> = body.split('/').filter(|s| !s.is_empty()).collect(); + // Not filtered. Dropping empty segments quietly accepted `/Event//EventData/Data` and + // `/Event/`, which are malformed paths that would then evaluate as though they were the + // path the author meant. A map with a typo should be reported, not silently reinterpreted. + let segments: Vec<&str> = body.split('/').collect(); for (index, segment) in segments.iter().enumerate() { + if segment.is_empty() { + // A single trailing slash is the one benign case: `/Event/EventData/` names the + // same element as without it. + if index + 1 == segments.len() && !steps.is_empty() { + continue; + } + return Err(PathError::MalformedStep { + path: expression.to_string(), + step: String::new(), + }); + } if let Some(attribute_name) = segment.strip_prefix('@') { if index + 1 != segments.len() { return Err(PathError::MisplacedAttribute(expression.to_string())); } + if attribute_name.is_empty() { + return Err(PathError::MalformedStep { + path: expression.to_string(), + step: segment.to_string(), + }); + } attribute = Some(attribute_name.to_string()); continue; } @@ -418,4 +458,35 @@ mod tests { let path = ValuePath::parse("/Event/EventData/Data/@Name").expect("parses"); assert_eq!(path.evaluate(&event).as_deref(), Some("first")); } + + #[test] + fn a_doubled_separator_is_refused_rather_than_reinterpreted() { + // `/Event//EventData/Data` is a typo. Skipping the empty segment would read it as + // `/Event/EventData/Data` and apply a map the author did not write. + assert!(matches!( + ValuePath::parse("/Event//EventData/Data"), + Err(PathError::MalformedStep { .. }) + )); + } + + #[test] + fn an_attribute_with_no_name_is_refused() { + assert!(matches!( + ValuePath::parse("/Event/@"), + Err(PathError::MalformedStep { .. }) + )); + } + + #[test] + fn a_single_trailing_slash_names_the_same_element() { + // The one benign empty segment: `/Event/EventData/` selects what `/Event/EventData` does. + let with = ValuePath::parse("/Event/EventData/").expect("parses"); + let without = ValuePath::parse("/Event/EventData").expect("parses"); + assert_eq!(with, without); + } + + #[test] + fn a_bare_slash_is_still_an_empty_path() { + assert!(matches!(ValuePath::parse("/"), Err(PathError::Empty(_)))); + } } diff --git a/src-tauri/src/event_log/provider_db.rs b/src-tauri/src/event_log/provider_db.rs index 6ecf86d08..d8051bb79 100644 --- a/src-tauri/src/event_log/provider_db.rs +++ b/src-tauri/src/event_log/provider_db.rs @@ -268,13 +268,23 @@ impl ProviderStore { } } - if let Ok(mut open) = self.open_databases.lock() { - *open = databases; - } + // The locks are taken before anything is published, and a failure aborts rather than + // continuing. This takes &mut self, so neither lock can be contended and the only failure + // is poisoning; swallowing it left `info` describing databases that were never opened, so + // registered() reported coverage that no lookup could deliver. + let mut open = self + .open_databases + .lock() + .map_err(|_| "provider store lock was poisoned".to_string())?; + let mut cache = self + .cache + .lock() + .map_err(|_| "provider cache lock was poisoned".to_string())?; + *open = databases; + cache.clear(); + drop(open); + drop(cache); self.info = info.clone(); - if let Ok(mut cache) = self.cache.lock() { - cache.clear(); - } if info.is_empty() && !failures.is_empty() { return Err(failures.join("; ")); diff --git a/src/workspaces/event-log/EvtxFilterBar.tsx b/src/workspaces/event-log/EvtxFilterBar.tsx index bb3076e7a..bc3caa713 100644 --- a/src/workspaces/event-log/EvtxFilterBar.tsx +++ b/src/workspaces/event-log/EvtxFilterBar.tsx @@ -311,9 +311,8 @@ export function EvtxFilterBar() { style={{ minWidth: "104px" }} title="Choose which columns the list shows." onOptionSelect={(_, data) => { - const id = data.optionValue as EvtxColumnId | "__reset__"; - if (id === "__reset__") resetColumns(); - else if (id) toggleColumnVisible(id); + const id = data.optionValue as EvtxColumnId | undefined; + if (id) toggleColumnVisible(id); }} > {choosableColumns.map((column) => ( @@ -321,11 +320,23 @@ export function EvtxFilterBar() { {column.label} ))} - + {/* + A button, not an option. In a multiselect listbox every option carries a selection + indicator, so "Reset to defaults" rendered as though it were a column that could be + checked, and it is an action rather than a member of the set. + */} + + {/* Reordering lives outside the listbox. Buttons nested in a Fluent Option are invalid ARIA and never receive focus, because a listbox moves focus between options rather than into diff --git a/src/workspaces/event-log/EvtxTimeline.tsx b/src/workspaces/event-log/EvtxTimeline.tsx index 3e97cf6e9..1389d3550 100644 --- a/src/workspaces/event-log/EvtxTimeline.tsx +++ b/src/workspaces/event-log/EvtxTimeline.tsx @@ -13,6 +13,7 @@ import { type EvtxRow, } from "./evtx-filter"; import type { EvtxRecord, EvtxLevel } from "./types"; +import { visibleColumns } from "./evtx-columns"; import { EvtxTimelineRow } from "./EvtxTimelineRow"; const LEVEL_ORDER: Record = { @@ -114,6 +115,11 @@ export function EvtxTimeline() { // Grouping produces header rows interleaved with records, so the virtualizer indexes rows rather // than records. With no grouping the row list is the record list and nothing changes. + // Computed once rather than per row: columnConfig is stable between renders, and the row + // renderer was rebuilding the spec array and re-synthesizing every map column spec for each of + // potentially a hundred thousand rows. + const columns = useMemo(() => visibleColumns(columnConfig), [columnConfig]); + const rows: EvtxRow[] = useMemo( () => buildGroupedRows(sortedRecords, groupBy, collapsedGroups, timeZoneMode), [sortedRecords, groupBy, collapsedGroups, timeZoneMode] @@ -222,7 +228,10 @@ export function EvtxTimeline() { return (
0 ? "tree" : "listbox"} tabIndex={0} onKeyDown={handleKeyDown} aria-label={`Event log timeline - ${sortedRecords.length} records`} @@ -261,7 +270,8 @@ export function EvtxTimeline() { key={virtualRow.key} ref={virtualizer.measureElement} data-index={virtualRow.index} - role="button" + role="treeitem" + aria-level={row.depth + 1} // Focusable and activated by keyboard. It was reachable only by pointer, so a // keyboard user could not expand or collapse a group at all. tabIndex={0} @@ -313,6 +323,7 @@ export function EvtxTimeline() { monoFontSize={monoFontSize} lineHeight={lineHeight} columnConfig={columnConfig} + columns={columns} timeZoneMode={timeZoneMode} onSelect={setSelectedRecordId} /> diff --git a/src/workspaces/event-log/EvtxTimelineRow.tsx b/src/workspaces/event-log/EvtxTimelineRow.tsx index 7b3e60342..7a10af21b 100644 --- a/src/workspaces/event-log/EvtxTimelineRow.tsx +++ b/src/workspaces/event-log/EvtxTimelineRow.tsx @@ -7,8 +7,8 @@ import type { EvtxRecord, EvtxLevel } from "./types"; import { columnValue, columnWidth, - visibleColumns, type EvtxColumnConfig, + type EvtxColumnSpec, } from "./evtx-columns"; import type { EvtxTimeZoneMode } from "./evtx-time"; @@ -37,6 +37,8 @@ export interface EvtxTimelineRowProps { monoFontSize: number; lineHeight: string; columnConfig: EvtxColumnConfig; + /** Precomputed by the list, since columnConfig is stable and this renders once per row. */ + columns: EvtxColumnSpec[]; /** Passed rather than read from the store, so a memoized row re-renders when the clock changes. */ timeZoneMode: EvtxTimeZoneMode; onSelect: (id: number | null) => void; @@ -53,6 +55,7 @@ export const EvtxTimelineRow = memo( monoFontSize, lineHeight, columnConfig, + columns, timeZoneMode, onSelect, }, @@ -94,7 +97,7 @@ export const EvtxTimelineRow = memo( minWidth: 0, }} > - {visibleColumns(columnConfig).map((column) => { + {columns.map((column) => { const width = columnWidth(columnConfig, column); const value = columnValue(record, column.id, timeZoneMode); diff --git a/src/workspaces/event-log/evtx-store-coverage.test.ts b/src/workspaces/event-log/evtx-store-coverage.test.ts index e9198ec0a..339b902ec 100644 --- a/src/workspaces/event-log/evtx-store-coverage.test.ts +++ b/src/workspaces/event-log/evtx-store-coverage.test.ts @@ -138,4 +138,23 @@ describe("the time window reaches the service", () => { expect(filterOf(0)?.time).toBeUndefined(); }); + + it("drops a non-string gap the reader sent rather than rendering it", () => { + // The guard normalizes, but only if callers use what it returns. Ignoring the return value + // left the raw reply in place and stored 42 in coverageGaps. + invoke.mockResolvedValueOnce({ + records: [], + channels: [], + totalRecords: 0, + parseErrors: 1, + errorMessages: ["real gap", 42, null], + }); + + return useEvtxStore + .getState() + .queryChannels(["Application"]) + .then(() => { + expect(useEvtxStore.getState().coverageGaps).toEqual(["real gap"]); + }); + }); }); diff --git a/src/workspaces/event-log/evtx-store.ts b/src/workspaces/event-log/evtx-store.ts index 8f827a84e..e8ac0cb41 100644 --- a/src/workspaces/event-log/evtx-store.ts +++ b/src/workspaces/event-log/evtx-store.ts @@ -153,8 +153,8 @@ export const useEvtxStore = create()((set, get) => ({ set({ isLoading: true, loadError: null }); try { const result = await invoke("evtx_parse_files", { paths }); - assertParseResultShape(result); - set(applyParseResult(result, "files")); + const checked = assertParseResultShape(result); + set(applyParseResult({ ...result, errorMessages: checked.errorMessages }, "files")); } catch (error) { const message = error instanceof Error ? error.message : String(error); set({ isLoading: false, loadError: message }); @@ -194,7 +194,7 @@ export const useEvtxStore = create()((set, get) => ({ }); // Query all core channels in parallel (bypass queryChannels to avoid isLoading conflicts) - const mergeResult = (ch: string, result: EvtxParseResult) => { + const mergeResult = (ch: string, result: EvtxParseResult, gaps: string[]) => { const state = get(); const merged = [...state.records, ...result.records]; merged.sort((a, b) => a.timestampEpoch - b.timestampEpoch); @@ -216,10 +216,7 @@ export const useEvtxStore = create()((set, get) => ({ // Channels load one at a time and each may report its own gaps, so they accumulate // rather than replace. Deduplicated because re-querying a channel would otherwise // repeat the same line. - coverageGaps: mergeCoverageGaps( - state.coverageGaps, - result.errorMessages - ), + coverageGaps: mergeCoverageGaps(state.coverageGaps, gaps), }); }; @@ -230,8 +227,8 @@ export const useEvtxStore = create()((set, get) => ({ maxEvents: null, filter: buildServerFilter(get().timeWindow), }); - assertParseResultShape(result); - mergeResult(ch, result); + const checked = assertParseResultShape(result); + mergeResult(ch, result, checked.errorMessages); } catch (e) { const msg = e instanceof Error ? e.message : String(e); console.warn(`[evtx] Failed to query ${ch}: ${msg}`); @@ -262,7 +259,7 @@ export const useEvtxStore = create()((set, get) => ({ maxEvents: maxEvents ?? null, filter: buildServerFilter(get().timeWindow), }); - assertParseResultShape(result); + const checked = assertParseResultShape(result); // Merge new records with existing ones (for incremental channel loading) const state = get(); @@ -292,7 +289,7 @@ export const useEvtxStore = create()((set, get) => ({ loadError: null, // Accumulated, not dropped. This path loads channels incrementally, so discarding what the // backend reported here would show a complete view of a partly unreadable set. - coverageGaps: mergeCoverageGaps(state.coverageGaps, result.errorMessages), + coverageGaps: mergeCoverageGaps(state.coverageGaps, checked.errorMessages), selectedRecordId: null, }); } catch (error) { @@ -338,7 +335,7 @@ export const useEvtxStore = create()((set, get) => ({ // outside the window the toolbar was still showing as selected. filter: buildServerFilter(get().timeWindow), }); - assertParseResultShape(result); + const checked = assertParseResultShape(result); const s = get(); const merged = [...s.records, ...result.records]; @@ -358,7 +355,7 @@ export const useEvtxStore = create()((set, get) => ({ channels: newChannels, loadedChannels: newLoaded, loadElapsedMs: performance.now() - startTime, - coverageGaps: mergeCoverageGaps(s.coverageGaps, result.errorMessages), + coverageGaps: mergeCoverageGaps(s.coverageGaps, checked.errorMessages), }); } catch (e) { console.warn(`[evtx] Refresh failed for ${ch}:`, e); From 6c546d56ef8b68acdec0b59038700d5d706b1db7 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 10 Aug 2026 15:19:12 -0400 Subject: [PATCH 66/85] fix(event-log): clear a selection collapse hides, and stop two tests passing vacuously Collapsing a group left the selected record in sortedRecords while taking its row out of the list, so keyboard navigation could not find the current position and the detail pane showed a record that was not on screen. The visibility check now runs against the rendered rows, which covers filtering as well, so the two effects become one. Two tests could not fail for their stated reason, both mine: - the non-finite lastUsed test built its input with JSON.stringify, which writes 1e309 as null, so the assertion held whether or not the sanitizer rejected Infinity. It uses literal JSON text now, and removing the guard fails it. - the unknown-event-id test ran against an empty store, so the lookup returned None on the provider and the event-id branch was never reached; it only repeated the no-database case. It needs a store that holds the provider, so it is gated on CMTRACEOPEN_PROVIDER_DB like its siblings. Co-Authored-By: Claude Opus 5 --- src-tauri/src/event_log/parser.rs | 16 ++++++++++++++- src/workspaces/event-log/EvtxTimeline.tsx | 20 +++++++++++-------- .../event-log/evtx-saved-filters.test.ts | 4 +++- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src-tauri/src/event_log/parser.rs b/src-tauri/src/event_log/parser.rs index e80e14266..6f8e84c01 100644 --- a/src-tauri/src/event_log/parser.rs +++ b/src-tauri/src/event_log/parser.rs @@ -713,9 +713,23 @@ mod description_tests { } #[test] + #[ignore = "requires a real provider database via CMTRACEOPEN_PROVIDER_DB"] fn an_unknown_event_id_falls_back_rather_than_inventing_a_description() { + // Needs a store that actually holds the provider. Against an empty one the lookup returns + // None on the provider itself and the event-id branch is never reached, so this only + // repeated the no-database case while claiming to cover something else. + let store = loaded_store(); let data = insertions(&[("X", "1")]); - assert!(describe_event(&empty_store(), "Still-Not-Loaded", 999_999, &data).is_none()); + assert!( + describe_event( + &store, + "Microsoft-Windows-DeviceManagement-Enterprise-Diagnostics-Provider", + 999_999, + &data + ) + .is_none(), + "a provider that is loaded but does not define this id must fall back" + ); } #[test] diff --git a/src/workspaces/event-log/EvtxTimeline.tsx b/src/workspaces/event-log/EvtxTimeline.tsx index 1389d3550..bf9820755 100644 --- a/src/workspaces/event-log/EvtxTimeline.tsx +++ b/src/workspaces/event-log/EvtxTimeline.tsx @@ -103,14 +103,6 @@ export function EvtxTimeline() { ); }, [filteredRecords, sortField, sortDirection]); - useEffect(() => { - if (selectedRecordId == null) return; - const stillVisible = sortedRecords.some((r) => r.id === selectedRecordId); - if (!stillVisible) { - setSelectedRecordId(null); - } - }, [sortedRecords, setSelectedRecordId, selectedRecordId]); - const parentRef = useRef(null); // Grouping produces header rows interleaved with records, so the virtualizer indexes rows rather @@ -136,6 +128,18 @@ export function EvtxTimeline() { [rows] ); + // Checked against the rendered rows, not the filtered records. Filtering already dropped a + // hidden selection, but collapsing a group leaves the record in sortedRecords while taking its + // row out of the list, so keyboard navigation could not find the current position and the + // detail pane showed a record that was not on screen. Rows covers both. + useEffect(() => { + if (selectedRecordId === null) return; + const stillVisible = rows.some( + (row) => row.kind === "record" && row.record.id === selectedRecordId + ); + if (!stillVisible) setSelectedRecordId(null); + }, [rows, selectedRecordId, setSelectedRecordId]); + const virtualizer = useVirtualizer({ count: rows.length, getScrollElement: () => parentRef.current, diff --git a/src/workspaces/event-log/evtx-saved-filters.test.ts b/src/workspaces/event-log/evtx-saved-filters.test.ts index f6f84cbb8..1c35850cb 100644 --- a/src/workspaces/event-log/evtx-saved-filters.test.ts +++ b/src/workspaces/event-log/evtx-saved-filters.test.ts @@ -153,8 +153,10 @@ describe("hostile stored values", () => { it("rejects a non-finite lastUsed", () => { // JSON admits 1e309, which parses to Infinity and would reach the ordering comparator as a // non-finite operand. + // Literal JSON text, not JSON.stringify: stringify writes 1e309 as null, so the assertion + // held whether or not the sanitizer rejected Infinity. JSON.parse produces Infinity. const parsed = parseFilterExport( - JSON.stringify({ schema: 1, filters: [{ id: "a", name: "A", lastUsed: 1e309 }] }) + '{"schema":1,"filters":[{"id":"a","name":"A","lastUsed":1e309}]}' ); expect(parsed.filters).toHaveLength(1); expect(parsed.filters[0].lastUsed).toBeNull(); From d3d18dc4943e5c380e5fdfedc594ec45108d51dd Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 11 Aug 2026 12:13:46 -0400 Subject: [PATCH 67/85] perf(eventmap): parse each map's paths once instead of once per record Applying a map runs for every event on a channel, and for each binding it rebuilt the `%Name%` placeholder and re-parsed the path expression. Both are constant for the life of the map, so a million-record scan did a million identical parses. Measured against the Security 4624 fixture, re-parsing versus compiled: | | per record | per 10k records | |----------------------|------------|-----------------| | re-parsing each time | 5.20 us | 49.0 ms | | compiled once | 1.91 us | 18.9 ms | 2.6x, and it scales with channel size rather than being a fixed saving. Memoized on MapEntry behind a OnceLock rather than compiled by whoever owns the map. A separate compile step is one a caller can forget, and a map that skipped it would resolve nothing while looking exactly like a map that did not match, which is the failure mode this workspace keeps having to design against. There is no uncompiled state to reach. A malformed expression is kept as a None path rather than dropped, so the applier still reports it: a typo in a map is a defect an operator needs told about, not a binding that silently resolves to nothing. MapEntry gains a constructor, since the cache is a private field, and hand implements PartialEq over the deserialized content so two entries describing the same mapping are equal whether or not either has been applied. The parser crate had no benchmark harness; criterion is a dev-dependency, so it never reaches the wasm32 build of the library. Closes #544. Co-Authored-By: Claude Opus 5 --- Cargo.lock | 1 + crates/cmtraceopen-parser/Cargo.toml | 6 + .../benches/eventmap_apply.rs | 87 ++++++++++++ .../cmtraceopen-parser/src/eventmap/apply.rs | 133 +++++++++--------- .../cmtraceopen-parser/src/eventmap/model.rs | 73 +++++++++- 5 files changed, 233 insertions(+), 67 deletions(-) create mode 100644 crates/cmtraceopen-parser/benches/eventmap_apply.rs diff --git a/Cargo.lock b/Cargo.lock index f248063a8..0806e51fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -631,6 +631,7 @@ version = "0.1.1" dependencies = [ "base64 0.22.1", "chrono", + "criterion", "encoding_rs", "log", "regex", diff --git a/crates/cmtraceopen-parser/Cargo.toml b/crates/cmtraceopen-parser/Cargo.toml index 1acf9c263..17126e925 100644 --- a/crates/cmtraceopen-parser/Cargo.toml +++ b/crates/cmtraceopen-parser/Cargo.toml @@ -29,3 +29,9 @@ base64 = "0.22" sha2 = "0.11" [dev-dependencies] +# Benchmarks only. A dev-dependency, so it never reaches the wasm32 build of the library. +criterion = "0.8" + +[[bench]] +name = "eventmap_apply" +harness = false diff --git a/crates/cmtraceopen-parser/benches/eventmap_apply.rs b/crates/cmtraceopen-parser/benches/eventmap_apply.rs new file mode 100644 index 000000000..0968819d3 --- /dev/null +++ b/crates/cmtraceopen-parser/benches/eventmap_apply.rs @@ -0,0 +1,87 @@ +//! How long applying one map to one event takes. +//! +//! The map engine runs once per record, so this is multiplied by the channel size: a million-record +//! scan pays it a million times. The benchmark exists because the path expressions and `%Name%` +//! placeholders in a map are constant for its whole life, and the applier used to rebuild both for +//! every record. +//! +//! Measured on an M-series mac against the Security 4624 fixture, comparing re-parsing per record +//! against the compiled cache: +//! +//! | | per record | per 10k records | +//! |---|---|---| +//! | re-parsing each time | 5.20 us | 49.0 ms | +//! | compiled once | 1.91 us | 18.9 ms | +//! +//! Run with `cargo bench --bench eventmap_apply` from `crates/cmtraceopen-parser`. + +use criterion::{criterion_group, criterion_main, Criterion}; +use std::hint::black_box; + +use cmtraceopen_parser::eventmap::{apply_map, EventMap, EventNode}; + +/// A map shaped like the upstream Security 4624: several bindings, multi-placeholder templates. +const SECURITY_4624: &str = include_str!("../tests/fixtures/eventmap/security-4624.json"); + +fn event() -> EventNode { + EventNode::new("Event").with_child( + EventNode::new("EventData") + .with_child( + EventNode::new("Data") + .with_attribute("Name", "SubjectUserName") + .with_text("adam"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "SubjectDomainName") + .with_text("CONTOSO"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "TargetUserName") + .with_text("svc-backup"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "IpAddress") + .with_text("10.0.0.7"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "LogonType") + .with_text("10"), + ), + ) +} + +fn apply_one_record(c: &mut Criterion) { + let map: EventMap = serde_json::from_str(SECURITY_4624).expect("fixture parses"); + let event = event(); + + // Warm the compiled cache the way a real scan does: the first record pays for the parse, and + // every record after it reuses the result. Measuring from cold would report the one-off cost + // rather than the per-record cost that actually multiplies. + let _ = apply_map(&map, &event); + + c.bench_function("apply_map/security-4624/one record", |b| { + b.iter(|| black_box(apply_map(black_box(&map), black_box(&event)))); + }); +} + +fn apply_a_channel(c: &mut Criterion) { + let map: EventMap = serde_json::from_str(SECURITY_4624).expect("fixture parses"); + let event = event(); + let _ = apply_map(&map, &event); + + // Ten thousand records is a small channel; the point is the shape of the curve, not the total. + c.bench_function("apply_map/security-4624/10k records", |b| { + b.iter(|| { + for _ in 0..10_000 { + black_box(apply_map(black_box(&map), black_box(&event))); + } + }); + }); +} + +criterion_group!(benches, apply_one_record, apply_a_channel); +criterion_main!(benches); diff --git a/crates/cmtraceopen-parser/src/eventmap/apply.rs b/crates/cmtraceopen-parser/src/eventmap/apply.rs index 26c691ced..9b8a2393c 100644 --- a/crates/cmtraceopen-parser/src/eventmap/apply.rs +++ b/crates/cmtraceopen-parser/src/eventmap/apply.rs @@ -8,7 +8,6 @@ use std::collections::BTreeMap; use super::model::{EventMap, MapEntry, MapProperty}; use super::node::EventNode; -use super::path::ValuePath; /// One normalized column produced from an event. #[derive(Debug, Clone, PartialEq, Eq)] @@ -74,15 +73,17 @@ fn apply_entry( let template = entry.property_value.as_str(); let mut resolved: BTreeMap<&str, String> = BTreeMap::new(); - for binding in &entry.values { + // Paths and placeholders are parsed once per map, not once per record. This runs for every + // event on a channel, so re-parsing an expression that cannot change was the dominant cost. + for (binding, compiled) in entry.values.iter().zip(entry.compiled()) { // Checked against the original template, never against partially rendered output. - if !template.contains(&format!("%{}%", binding.name)) { + if !template.contains(&compiled.placeholder) { continue; } - let value = match ValuePath::parse(&binding.value) { - Ok(path) => path.evaluate(event).map(|value| value.into_owned()), - Err(_) => { + let value = match &compiled.path { + Some(path) => path.evaluate(event).map(|value| value.into_owned()), + None => { let defect = (binding.name.clone(), binding.value.clone()); if !invalid_paths.contains(&defect) { invalid_paths.push(defect); @@ -210,14 +211,14 @@ mod tests { #[test] fn substitutes_multiple_placeholders_in_one_template() { let map = map_with( - vec![MapEntry { - property: MapProperty::UserName, - property_value: "%domain%\\%user%".to_string(), - values: vec![ + vec![MapEntry::new( + MapProperty::UserName, + "%domain%\\%user%".to_string(), + vec![ binding("domain", "SubjectDomainName"), binding("user", "SubjectUserName"), ], - }], + )], vec![], ); @@ -229,11 +230,11 @@ mod tests { #[test] fn missing_field_is_reported_and_leaves_the_placeholder_visible() { let map = map_with( - vec![MapEntry { - property: MapProperty::PayloadData(1), - property_value: "LogonType %LogonType%".to_string(), - values: vec![binding("LogonType", "LogonType")], - }], + vec![MapEntry::new( + MapProperty::PayloadData(1), + "LogonType %LogonType%".to_string(), + vec![binding("LogonType", "LogonType")], + )], vec![], ); @@ -247,11 +248,11 @@ mod tests { #[test] fn lookup_translates_a_bound_variable() { let map = map_with( - vec![MapEntry { - property: MapProperty::PayloadData(1), - property_value: "Bus: %BusType%".to_string(), - values: vec![binding("BusType", "BusType")], - }], + vec![MapEntry::new( + MapProperty::PayloadData(1), + "Bus: %BusType%".to_string(), + vec![binding("BusType", "BusType")], + )], vec![Lookup { name: "BusType".to_string(), default: Some("Unknown code".to_string()), @@ -269,11 +270,11 @@ mod tests { #[test] fn lookup_default_applies_to_an_unknown_code() { let map = map_with( - vec![MapEntry { - property: MapProperty::PayloadData(1), - property_value: "Bus: %BusType%".to_string(), - values: vec![binding("BusType", "BusType")], - }], + vec![MapEntry::new( + MapProperty::PayloadData(1), + "Bus: %BusType%".to_string(), + vec![binding("BusType", "BusType")], + )], vec![Lookup { name: "BusType".to_string(), default: Some("Unknown code".to_string()), @@ -291,11 +292,11 @@ mod tests { #[test] fn a_template_with_no_placeholders_is_emitted_verbatim() { let map = map_with( - vec![MapEntry { - property: MapProperty::PayloadData(1), - property_value: "Screen saver invoked".to_string(), - values: vec![], - }], + vec![MapEntry::new( + MapProperty::PayloadData(1), + "Screen saver invoked".to_string(), + vec![], + )], vec![], ); @@ -310,11 +311,11 @@ mod tests { #[test] fn a_placeholder_with_no_binding_is_reported_unresolved() { let map = map_with( - vec![MapEntry { - property: MapProperty::PayloadData(1), - property_value: "Value %NeverBound%".to_string(), - values: vec![], - }], + vec![MapEntry::new( + MapProperty::PayloadData(1), + "Value %NeverBound%".to_string(), + vec![], + )], vec![], ); @@ -325,14 +326,14 @@ mod tests { #[test] fn a_malformed_path_is_a_map_defect_not_a_missing_field() { let map = map_with( - vec![MapEntry { - property: MapProperty::PayloadData(1), - property_value: "%broken%".to_string(), - values: vec![ValueBinding { + vec![MapEntry::new( + MapProperty::PayloadData(1), + "%broken%".to_string(), + vec![ValueBinding { name: "broken".to_string(), value: "EventData/Data".to_string(), }], - }], + )], vec![], ); @@ -362,14 +363,14 @@ mod tests { ), ); let map = map_with( - vec![MapEntry { - property: MapProperty::UserName, - property_value: "%domain%\\%user%".to_string(), - values: vec![ + vec![MapEntry::new( + MapProperty::UserName, + "%domain%\\%user%".to_string(), + vec![ binding("domain", "SubjectDomainName"), binding("user", "SubjectUserName"), ], - }], + )], vec![], ); @@ -383,11 +384,11 @@ mod tests { #[test] fn a_literal_percent_does_not_hide_the_placeholder_that_follows_it() { let map = map_with( - vec![MapEntry { - property: MapProperty::PayloadData(1), - property_value: "50% off %Cost%".to_string(), - values: vec![], - }], + vec![MapEntry::new( + MapProperty::PayloadData(1), + "50% off %Cost%".to_string(), + vec![], + )], vec![], ); @@ -401,11 +402,11 @@ mod tests { #[test] fn a_literal_percent_survives_when_a_later_placeholder_resolves() { let map = map_with( - vec![MapEntry { - property: MapProperty::PayloadData(1), - property_value: "50% off for %user%".to_string(), - values: vec![binding("user", "SubjectUserName")], - }], + vec![MapEntry::new( + MapProperty::PayloadData(1), + "50% off for %user%".to_string(), + vec![binding("user", "SubjectUserName")], + )], vec![], ); @@ -420,11 +421,11 @@ mod tests { #[test] fn an_unpaired_trailing_percent_is_preserved_verbatim() { let map = map_with( - vec![MapEntry { - property: MapProperty::PayloadData(1), - property_value: "complete: 100%".to_string(), - values: vec![], - }], + vec![MapEntry::new( + MapProperty::PayloadData(1), + "complete: 100%".to_string(), + vec![], + )], vec![], ); @@ -439,14 +440,14 @@ mod tests { #[test] fn an_unused_binding_does_not_affect_the_result() { let map = map_with( - vec![MapEntry { - property: MapProperty::PayloadData(1), - property_value: "User %user%".to_string(), - values: vec![ + vec![MapEntry::new( + MapProperty::PayloadData(1), + "User %user%".to_string(), + vec![ binding("user", "SubjectUserName"), binding("unused", "DoesNotExist"), ], - }], + )], vec![], ); diff --git a/crates/cmtraceopen-parser/src/eventmap/model.rs b/crates/cmtraceopen-parser/src/eventmap/model.rs index c43945fa2..8c4c418d9 100644 --- a/crates/cmtraceopen-parser/src/eventmap/model.rs +++ b/crates/cmtraceopen-parser/src/eventmap/model.rs @@ -6,9 +6,12 @@ //! of the parser crate and lets tests drive the real corpus through `serde_json`. use std::collections::BTreeMap; +use std::sync::OnceLock; use serde::{Deserialize, Deserializer}; +use super::path::ValuePath; + /// A normalized output column that a map entry writes into. /// /// The upstream corpus is not perfectly consistent: four maps spell the target `Username` rather @@ -99,8 +102,28 @@ impl Lookup { } } +/// A binding with its path expression already parsed and its placeholder already formatted. +/// +/// Both are constant for the life of the map, and applying a map happens once per record. Parsing +/// the same expression a million times to get a million identical results is the cost this avoids. +#[derive(Debug, Clone)] +pub struct CompiledBinding { + /// `%Name%`, formatted once. + pub placeholder: String, + /// The parsed path, or `None` when the expression is malformed. + /// + /// A malformed expression is kept as a value rather than dropped so the applier can still + /// report it: a map with a typo in it is a defect an operator needs told about, not a binding + /// that silently resolves to nothing. + pub path: Option, +} + /// One output column produced from an event. -#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +/// +/// `PartialEq` compares the deserialized content only. The compiled cache below is derived from +/// it, so two entries that describe the same mapping are equal whether or not either has been +/// applied yet. +#[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct MapEntry { /// Which normalized column this writes. @@ -110,8 +133,56 @@ pub struct MapEntry { /// Variables available to the template. #[serde(default)] pub values: Vec, + /// Built on first use and reused for every later record. + /// + /// Memoized here rather than compiled by whoever owns the map, because a separate compile step + /// is one a caller can forget: a map that skipped it would resolve nothing and look like a map + /// that simply did not match. + #[serde(skip)] + compiled: OnceLock>, +} + +impl MapEntry { + /// Builds an entry. + /// + /// The compiled cache is not a constructor argument: it is derived from these fields and is + /// built on first use, so there is no state a caller could supply inconsistently. + pub fn new(property: MapProperty, property_value: String, values: Vec) -> Self { + Self { + property, + property_value, + values, + compiled: OnceLock::new(), + } + } + + /// The entry's bindings, parsed once. + /// + /// Only bindings whose placeholder appears in the template are compiled, matching what the + /// applier will actually consult; the rest cost nothing. + pub fn compiled(&self) -> &[CompiledBinding] { + self.compiled.get_or_init(|| { + self.values + .iter() + .map(|binding| CompiledBinding { + placeholder: format!("%{}%", binding.name), + path: ValuePath::parse(&binding.value).ok(), + }) + .collect() + }) + } } +impl PartialEq for MapEntry { + fn eq(&self, other: &Self) -> bool { + self.property == other.property + && self.property_value == other.property_value + && self.values == other.values + } +} + +impl Eq for MapEntry {} + /// A parsed EvtxECmd map file. /// /// Identity is `(channel, provider, event_id)`, not the file name, matching upstream. From 01e1d3de776afa0046e0cc8f13f0a01903400e12 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 11 Aug 2026 12:31:18 -0400 Subject: [PATCH 68/85] fix(clipboard): let the mirror tests wait for the write they assert Two tests from #521 fail on main, which blocks every open PR. Verified on a clean origin/main checkout with no other branch involved. useClipboardHistoryMirror defers its plugin write with setTimeout so the native copy lands first, and that deferral is deliberate. The two tests that assert a write fired the copy event and asserted synchronously, so the timer had not run and writeText had zero calls. The negative cases need no timer, since a pending write would not make an assertion of "never called" pass. Fake timers scoped to that describe, advanced after the event. Verified the tests still have teeth: making the hook skip its write fails both. Touched from an event-log branch only because main being red blocks everything; the hook itself is unchanged. Co-Authored-By: Claude Opus 5 --- src/hooks/use-clipboard-copy.test.tsx | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/hooks/use-clipboard-copy.test.tsx b/src/hooks/use-clipboard-copy.test.tsx index df261e827..a843eac3b 100644 --- a/src/hooks/use-clipboard-copy.test.tsx +++ b/src/hooks/use-clipboard-copy.test.tsx @@ -162,11 +162,23 @@ describe("useKeyboard Ctrl+C fallback (#520)", () => { }); describe("useClipboardHistoryMirror (#520)", () => { + // The hook defers its write with setTimeout so the native copy lands first, so the two tests + // that assert a write have to let that timer run. The negative cases below need no timer: they + // assert the write never happens, and a pending timer would not change that. + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + it("mirrors a native copy of a DOM selection through the clipboard plugin", () => { stubSelection("part of the message"); renderHook(() => useClipboardHistoryMirror()); fireEvent.copy(document.body); + vi.runAllTimers(); expect(clipboardMocks.writeText).toHaveBeenCalledWith("part of the message"); }); @@ -180,6 +192,7 @@ describe("useClipboardHistoryMirror (#520)", () => { input.setSelectionRange(0, 6); fireEvent.copy(input); + vi.runAllTimers(); expect(clipboardMocks.writeText).toHaveBeenCalledWith("search"); input.remove(); From 4ad735db9f41b2f781cdc0cda9f53d95fa6d14c2 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 11 Aug 2026 20:50:19 -0400 Subject: [PATCH 69/85] fix(eventmap): make the memoized cache impossible to outlive its source Hermes charter re-review found a Major defect in the compiled-binding cache I added, and it is right. A OnceLock never invalidates, and MapEntry's property_value and values stayed public. After the first application a caller could change a binding's path, a binding's name, or the template, and apply the same entry again: compiled() returns the old records and apply_entry zips the current bindings against them. A changed path resolves through the old one, a changed name pairs a new binding with the old placeholder, and a changed template skips the new placeholder's binding and reports it unresolved. Same entry, different answer depending on whether it had been applied before. Both fields are private now, read through template() and bindings(). The alternative is an invalidation path, which is a thing to remember to call and therefore a thing to forget; this makes the divergence unrepresentable instead. Verified from a separate downstream crate that the field is genuinely unreachable rather than merely undocumented. The regression test Hermes asked for cannot be written as described, because mutating a source field no longer compiles. Two tests cover what remains: that applying one entry twice gives the same answer, and that a rebuilt entry resolves through its own expression rather than inheriting an earlier one's. Co-Authored-By: Claude Opus 5 --- .../cmtraceopen-parser/src/eventmap/apply.rs | 83 ++++++++++++++++++- .../cmtraceopen-parser/src/eventmap/model.rs | 25 +++++- .../tests/eventmap_corpus.rs | 2 +- 3 files changed, 105 insertions(+), 5 deletions(-) diff --git a/crates/cmtraceopen-parser/src/eventmap/apply.rs b/crates/cmtraceopen-parser/src/eventmap/apply.rs index 9b8a2393c..0b361663a 100644 --- a/crates/cmtraceopen-parser/src/eventmap/apply.rs +++ b/crates/cmtraceopen-parser/src/eventmap/apply.rs @@ -70,12 +70,12 @@ fn apply_entry( event: &EventNode, invalid_paths: &mut Vec<(String, String)>, ) -> MappedValue { - let template = entry.property_value.as_str(); + let template = entry.template(); let mut resolved: BTreeMap<&str, String> = BTreeMap::new(); // Paths and placeholders are parsed once per map, not once per record. This runs for every // event on a channel, so re-parsing an expression that cannot change was the dominant cost. - for (binding, compiled) in entry.values.iter().zip(entry.compiled()) { + for (binding, compiled) in entry.bindings().iter().zip(entry.compiled()) { // Checked against the original template, never against partially rendered output. if !template.contains(&compiled.placeholder) { continue; @@ -458,4 +458,83 @@ mod tests { ); assert!(mapped.values[0].is_complete()); } + + #[test] + fn applying_an_entry_twice_gives_the_same_answer() { + // The compiled bindings are memoized and a OnceLock never invalidates, so the risk is a + // cache that outlives the content it was built from. The inputs are private, which makes + // that unrepresentable rather than unlikely: there is no way to reach this entry's + // template or bindings and change them between these two calls. + let map = map_with( + vec![MapEntry::new( + MapProperty::PayloadData(1), + "%first% then %second%".to_string(), + vec![binding("first", "A"), binding("second", "B")], + )], + vec![], + ); + let event = EventNode::new("Event").with_child( + EventNode::new("EventData") + .with_child( + EventNode::new("Data") + .with_attribute("Name", "A") + .with_text("alpha"), + ) + .with_child( + EventNode::new("Data") + .with_attribute("Name", "B") + .with_text("beta"), + ), + ); + + let cold = apply_map(&map, &event); + let warm = apply_map(&map, &event); + assert_eq!(cold, warm); + assert_eq!( + warm.value_for(&MapProperty::PayloadData(1)), + Some("alpha then beta") + ); + } + + #[test] + fn a_rebuilt_entry_compiles_its_own_content() { + // Rebuilding is the only way to get different content now, so the guarantee that matters + // is that a new entry never inherits an older one's compiled paths. + let event = EventNode::new("Event").with_child( + EventNode::new("EventData").with_child( + EventNode::new("Data") + .with_attribute("Name", "A") + .with_text("alpha"), + ), + ); + + let first = map_with( + vec![MapEntry::new( + MapProperty::PayloadData(1), + "%only%".to_string(), + vec![binding("only", "A")], + )], + vec![], + ); + assert_eq!( + apply_map(&first, &event).value_for(&MapProperty::PayloadData(1)), + Some("alpha") + ); + + // Same placeholder, different path: the second entry must resolve by its own expression. + let second = map_with( + vec![MapEntry::new( + MapProperty::PayloadData(1), + "%only%".to_string(), + vec![binding("only", "missing")], + )], + vec![], + ); + let mapped = apply_map(&second, &event); + assert_eq!( + mapped.value_for(&MapProperty::PayloadData(1)), + Some("%only%"), + "the rebuilt entry must not resolve through the first entry's path" + ); + } } diff --git a/crates/cmtraceopen-parser/src/eventmap/model.rs b/crates/cmtraceopen-parser/src/eventmap/model.rs index 8c4c418d9..b5f1bace6 100644 --- a/crates/cmtraceopen-parser/src/eventmap/model.rs +++ b/crates/cmtraceopen-parser/src/eventmap/model.rs @@ -129,10 +129,18 @@ pub struct MapEntry { /// Which normalized column this writes. pub property: MapProperty, /// Template containing `%Name%` placeholders. - pub property_value: String, + /// + /// Private, with [`template`](Self::template) to read it. The compiled cache below is derived + /// from this field and from `values`, and a `OnceLock` does not invalidate: a caller who + /// mutated either after the first application would leave a complete but stale cache, and the + /// applier would pair current bindings with compiled records describing the old ones. Rather + /// than add an invalidation path nobody would remember to call, the inputs cannot be changed. + property_value: String, /// Variables available to the template. + /// + /// Private for the same reason as `property_value`. #[serde(default)] - pub values: Vec, + values: Vec, /// Built on first use and reused for every later record. /// /// Memoized here rather than compiled by whoever owns the map, because a separate compile step @@ -156,6 +164,19 @@ impl MapEntry { } } + /// The template this entry renders, with its `%Name%` placeholders. + pub fn template(&self) -> &str { + &self.property_value + } + + /// The variables the template can reference, in declaration order. + /// + /// Positionally aligned with [`compiled`](Self::compiled); both derive from the same field and + /// neither can change after construction. + pub fn bindings(&self) -> &[ValueBinding] { + &self.values + } + /// The entry's bindings, parsed once. /// /// Only bindings whose placeholder appears in the template are compiled, matching what the diff --git a/crates/cmtraceopen-parser/tests/eventmap_corpus.rs b/crates/cmtraceopen-parser/tests/eventmap_corpus.rs index c4f400c25..bf35c0ba4 100644 --- a/crates/cmtraceopen-parser/tests/eventmap_corpus.rs +++ b/crates/cmtraceopen-parser/tests/eventmap_corpus.rs @@ -226,7 +226,7 @@ fn every_fixture_parses_with_no_malformed_paths() { for raw in [SHELL_CORE_9701, SECURITY_4624, NTFS_146] { let map = load(raw); for entry in &map.maps { - for binding in &entry.values { + for binding in entry.bindings() { assert!( ValuePath::parse(&binding.value).is_ok(), "upstream map {} binding {} has an expression this engine cannot parse: {}", From d72096c94428a01ef667e1265365c5d1840e87b6 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 11 Aug 2026 21:32:21 -0400 Subject: [PATCH 70/85] docs(eventmap): say what compiled() actually does The doc claimed only bindings referenced by the template are compiled. Every binding is, and that is deliberate rather than an oversight: compiling all of them keeps the result positionally aligned with bindings(), which is what lets the applier zip the two. Skipping some would misalign the pair. The applier decides separately which to evaluate, and an unreferenced binding is never evaluated or reported. That is the behaviour the old wording was reaching for, attributed to the wrong function. Co-Authored-By: Claude Opus 5 --- crates/cmtraceopen-parser/src/eventmap/model.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/cmtraceopen-parser/src/eventmap/model.rs b/crates/cmtraceopen-parser/src/eventmap/model.rs index b5f1bace6..9d5deb3c3 100644 --- a/crates/cmtraceopen-parser/src/eventmap/model.rs +++ b/crates/cmtraceopen-parser/src/eventmap/model.rs @@ -179,8 +179,11 @@ impl MapEntry { /// The entry's bindings, parsed once. /// - /// Only bindings whose placeholder appears in the template are compiled, matching what the - /// applier will actually consult; the rest cost nothing. + /// Every binding is compiled, including any whose placeholder the template never references. + /// Compiling all of them keeps this positionally aligned with [`bindings`](Self::bindings), + /// which is what lets the applier zip the two; skipping some would misalign the pair. The + /// applier decides separately which to *evaluate*, and an unreferenced binding is never + /// evaluated or reported. pub fn compiled(&self) -> &[CompiledBinding] { self.compiled.get_or_init(|| { self.values From 282b90c19894a2661e7e85fb485f1d54aa26b88f Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 11 Aug 2026 22:40:15 -0400 Subject: [PATCH 71/85] fix(event-log): read the live path's fields from the parsed tree The live path scanned the rendered XML for itself: six System fields by substring search, and EventData by regular expression, on top of the parse it was already doing for the map engine. That regex required a Name attribute, could not match a value containing a newline, and never looked at UserData, so three classes of event field were dropped from the live view with nothing on screen indicating a field was missing. The file path never had those bugs because it read the tree, and it is the divergence between the two implementations that let them drift. The translation moves to event_log::rendered, which is not gated on Windows. It is pure - a string in, a record out - and gating it meant it could only be tested on the one platform that can produce its input, which is why it had no tests at all. extract_event_data moves next to extract_system_fields so both paths share one extractor. Also: the loop now parses each event once and hands the tree to the record builder rather than parsing again inside it; an event whose XML will not parse is counted and reported instead of pushed as a record with every field defaulted, which rendered as a real event at the epoch with no provider; and the message is only requested for an event that actually named a provider. Co-Authored-By: Claude Opus 5 --- src-tauri/src/event_log/event_node.rs | 85 ++++++ src-tauri/src/event_log/live.rs | 240 ++++------------- src-tauri/src/event_log/mod.rs | 1 + src-tauri/src/event_log/parser.rs | 85 +----- src-tauri/src/event_log/rendered.rs | 363 ++++++++++++++++++++++++++ 5 files changed, 498 insertions(+), 276 deletions(-) create mode 100644 src-tauri/src/event_log/rendered.rs diff --git a/src-tauri/src/event_log/event_node.rs b/src-tauri/src/event_log/event_node.rs index 72173d26a..869382a21 100644 --- a/src-tauri/src/event_log/event_node.rs +++ b/src-tauri/src/event_log/event_node.rs @@ -8,6 +8,8 @@ //! occasionally emit prefixed elements, but map path expressions are written without prefixes //! (`/Event/EventData/Data`), so keeping them would make every such map silently match nothing. +use super::models::EvtxField; +use super::sanitize_control_chars; use cmtraceopen_parser::eventmap::EventNode; use quick_xml::events::Event as XmlEvent; use quick_xml::{Reader, XmlVersion}; @@ -143,6 +145,7 @@ pub struct SystemFields { pub channel: Option, pub computer: Option, pub time_created: Option, + pub event_record_id: Option, pub task: Option, pub opcode: Option, pub process_id: Option, @@ -196,6 +199,7 @@ pub fn extract_system_fields(root: &EventNode) -> SystemFields { channel: text_of("Channel").map(str::to_string), computer: text_of("Computer").map(str::to_string), time_created: attribute_of("TimeCreated", "SystemTime"), + event_record_id: text_of("EventRecordID").and_then(|value| value.parse().ok()), task: text_of("Task").and_then(|value| value.parse().ok()), opcode: text_of("Opcode").and_then(|value| value.parse().ok()), process_id: attribute_of("Execution", "ProcessID").and_then(|v| v.parse().ok()), @@ -205,6 +209,87 @@ pub fn extract_system_fields(root: &EventNode) -> SystemFields { } } +/// What an event's data section yielded, in the two shapes that are needed. +/// +/// They differ, and conflating them corrupts messages. The display list omits fields the provider +/// left empty, because a column of blanks is noise. The insertion list keeps them, because the +/// message template addresses fields by position and a gap shifts every later reference. +pub struct EventFields { + pub fields: Vec, + pub insertions: Vec, +} + +/// Extract event fields as name-value pairs. +/// +/// Both `EventData` and `UserData` are read. Manifest providers use the former and classic or +/// trace-backed providers the latter, and skipping `UserData` would leave those events with no +/// fields at all. +/// +/// A `Data` element with no `Name` attribute is numbered by its position, matching how the event +/// message template refers to it. Values are sanitized to strip control characters that would +/// render as unexpected glyphs. +pub fn extract_event_data(root: &EventNode) -> EventFields { + let mut fields = Vec::new(); + let mut insertions = Vec::new(); + let mut unnamed = 0usize; + + let containers = root + .children + .iter() + .filter(|child| child.name == "EventData" || child.name == "UserData"); + + // A child that carries text is a field. A child that carries only elements is a wrapper, which + // is how UserData nests its fields under a provider-named element, so it is descended through. + // + // Deciding per child rather than per container matters: an EventData holding only has + // no at all, and treating the whole container as wrappers would descend into , + // find no children, and drop the only value the event carried. + let push_field = |child: &EventNode, + fields: &mut Vec, + insertions: &mut Vec, + unnamed: &mut usize| { + let value = sanitize_control_chars(child.text.as_deref().unwrap_or_default()); + // Recorded even when empty. The provider's message template addresses fields by position, + // so skipping one here shifts every later %N and renders the description with the wrong + // values substituted into it, which reads as fact. + insertions.push(value.clone()); + + // Counted before the emptiness check, for the same reason. The label is what an operator + // uses to match a field against the provider's template, so skipping the count for a blank + // slot would label the second field Data1 while the template calls it %2. + let position = if child.attribute("Name").is_none() && child.name == "Data" { + *unnamed += 1; + Some(*unnamed) + } else { + None + }; + + if value.is_empty() { + return; + } + let name = match (child.attribute("Name"), position) { + (Some(name), _) => name.to_string(), + (None, Some(position)) => format!("Data{position}"), + (None, None) => child.name.clone(), + }; + fields.push(EvtxField { name, value }); + }; + + for container in containers { + for child in &container.children { + if child.text.is_some() || child.children.is_empty() { + push_field(child, &mut fields, &mut insertions, &mut unnamed); + } else { + for grandchild in &child.children { + push_field(grandchild, &mut fields, &mut insertions, &mut unnamed); + } + } + } + } + + EventFields { fields, insertions } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src-tauri/src/event_log/live.rs b/src-tauri/src/event_log/live.rs index 7978de14e..380c97352 100644 --- a/src-tauri/src/event_log/live.rs +++ b/src-tauri/src/event_log/live.rs @@ -1,11 +1,8 @@ use std::collections::HashMap; use std::ffi::c_void; -use std::sync::OnceLock; -use regex::Regex; - -use super::models::{ChannelSourceType, EvtxChannelInfo, EvtxField, EvtxLevel, EvtxRecord}; -use super::{parse_timestamp_to_epoch_ms, sanitize_control_chars}; +use super::event_node::{extract_system_fields, parse_event_xml}; +use super::models::{ChannelSourceType, EvtxChannelInfo, EvtxRecord}; use cmtraceopen_parser::event_query::{build_query, EventQueryFilter}; use cmtraceopen_parser::eventmap::MapRegistry; @@ -210,6 +207,7 @@ fn query_channel_inner( let mut records = Vec::new(); let mut publisher_metadata = HashMap::>::new(); + let mut unparsable = 0usize; while records.len() < limit { let mut raw_handles = [0isize; EVENT_FETCH_BATCH]; @@ -247,35 +245,56 @@ fn query_channel_inner( let xml = render_event_xml(event_handle.raw()).map_err(|e| format_error("EvtRender", &e))?; - let provider_name = extract_xml_attr(&xml, "Provider", "Name"); + // Parsed once here and handed to the record builder. The provider has to be known + // before the record exists, because it names the publisher whose message template the + // service is asked to render, and parsing a second time to learn the rest would double + // the cost of the hottest loop in this view. + let parsed = match parse_event_xml(&xml) { + Ok(parsed) => parsed, + Err(error) => { + unparsable += 1; + if unparsable == 1 { + log::warn!( + "event=evtx_parse_failed channel=\"{channel}\" error=\"{error}\" xml_prefix=\"{}\"", + &xml[..xml.len().min(300)] + ); + } + continue; + } + }; + let system = extract_system_fields(&parsed); - // Try to get a formatted message via EvtFormatMessage - let rendered_message = provider_name.as_deref().and_then(|provider| { + // Only attempted when the event named a provider. Asking the service for the metadata + // of a publisher the event never named would fail once per event and cache the failure + // under a name no provider has. + let rendered_message = system.provider.as_deref().and_then(|provider| { format_event_message(event_handle.raw(), provider, &mut publisher_metadata) .ok() .flatten() }); - if let Some(record) = - parse_xml_to_record(&xml, channel, maps, rendered_message.as_deref()) - { - records.push(record); - // Report progress every 100 records - if records.len() % 100 == 0 { - on_progress(records.len(), None); - } - } else if records.is_empty() { - // Log the first unparseable XML so we can debug the format - log::warn!( - "event=evtx_parse_failed channel=\"{channel}\" xml_prefix=\"{}\"", - &xml[..xml.len().min(300)] - ); + records.push(super::rendered::record_from_parts( + &parsed, + system, + &xml, + channel, + maps, + rendered_message.as_deref(), + )); + // Report progress every 100 records + if records.len() % 100 == 0 { + on_progress(records.len(), None); } } } + if unparsable > 0 { + // Counted and reported rather than passed over. Events that never arrived look exactly like + // evidence that the thing being investigated did not happen. + log::warn!("event=evtx_live_query_gap channel=\"{channel}\" unparsable={unparsable}"); + } log::info!( - "event=evtx_live_query_done channel=\"{channel}\" records={}", + "event=evtx_live_query_done channel=\"{channel}\" records={} unparsable={unparsable}", records.len() ); Ok(records) @@ -414,178 +433,6 @@ fn format_event_message( } } -// ── XML parsing helpers ───────────────────────────────────────────────────── - -/// Parse rendered event XML into an EvtxRecord. -fn parse_xml_to_record( - xml: &str, - channel: &str, - maps: &MapRegistry, - rendered_message: Option<&str>, -) -> Option { - let event_id_str = extract_xml_text(xml, "EventID").unwrap_or_default(); - let event_id: u32 = event_id_str.parse().unwrap_or(0); - - let level_str = extract_xml_text(xml, "Level").unwrap_or_default(); - let level_val: u8 = level_str.parse().unwrap_or(4); - let level = EvtxLevel::from_level_value(level_val); - - let provider = extract_xml_attr(xml, "Provider", "Name").unwrap_or_default(); - let computer = extract_xml_text(xml, "Computer").unwrap_or_default(); - let event_record_id: u64 = extract_xml_text(xml, "EventRecordID") - .and_then(|s| s.parse().ok()) - .unwrap_or(0); - - let timestamp = extract_xml_attr(xml, "TimeCreated", "SystemTime").unwrap_or_default(); - let timestamp_epoch = parse_timestamp_to_epoch_ms(×tamp); - - let mut event_data = extract_xml_event_data(xml); - - // Parsed once and used for the System block, the decoded payload, and any registered map, so - // none of them costs an extra parse. - let parsed = super::event_node::parse_event_xml(xml).ok(); - - // Trace-backed channels carry their message as a hex blob rather than as EventData, so without - // this the row reads as a wall of hex digits. Surfaced as a field of its own because the raw - // hex never appears in EventData. - let payload = parsed - .as_ref() - .and_then(cmtraceopen_parser::event_payload::decode_payload_in) - .map(|decoded| sanitize_control_chars(&decoded.text)); - if let Some(text) = &payload { - event_data.push(EvtxField { - name: "EventPayload".to_string(), - value: text.clone(), - }); - } - - // The provider's own rendered message wins when there is one. A decoded payload comes next, - // used whole rather than through the summary, which would truncate the only text the event has - // at 80 characters. Sanitized to strip control characters that render as unexpected glyphs. - let message = rendered_message - .map(sanitize_control_chars) - .or(payload) - .unwrap_or_else(|| build_event_data_summary(&event_data)); - - let system = parsed - .as_ref() - .map(super::event_node::extract_system_fields) - .unwrap_or_default(); - let mapped = parsed - .as_ref() - .map(|root| super::maps::apply_registered(maps, channel, &provider, event_id, root)) - .unwrap_or_default(); - - Some(EvtxRecord { - id: 0, // assigned by commands.rs after sorting - event_record_id, - timestamp, - timestamp_epoch, - provider, - channel: channel.to_string(), - event_id, - level, - computer, - message, - event_data, - raw_xml: xml.to_string(), - source_label: "Live".to_string(), - task: system.task, - opcode: system.opcode, - process_id: system.process_id, - thread_id: system.thread_id, - user_sid: system.user_sid, - keywords: system.keywords, - mapped, - }) -} - -/// Extract an attribute value from an XML tag. -/// e.g. `extract_xml_attr(xml, "Provider", "Name")` for `` -fn extract_xml_attr(xml: &str, tag: &str, attr: &str) -> Option { - let tag_start = xml.find(&format!("<{tag}"))?; - let tag_end = xml[tag_start..].find('>')? + tag_start; - let tag_content = &xml[tag_start..=tag_end]; - - // Try both quote styles: Name='value' and Name="value" - for quote in ['"', '\''] { - let pattern = format!("{attr}={quote}"); - if let Some(attr_start) = tag_content.find(&pattern) { - let value_start = attr_start + pattern.len(); - if let Some(value_end) = tag_content[value_start..].find(quote) { - return Some(tag_content[value_start..value_start + value_end].to_string()); - } - } - } - None -} - -/// Extract text content between XML tags. -/// e.g. `extract_xml_text(xml, "EventID")` for `123` -fn extract_xml_text(xml: &str, tag: &str) -> Option { - let open = format!("<{tag}>"); - let open_with_attrs = format!("<{tag} "); - let close = format!(""); - - // Try simple value - if let Some(start) = xml.find(&open) { - let value_start = start + open.len(); - if let Some(end) = xml[value_start..].find(&close) { - return Some(xml[value_start..value_start + end].trim().to_string()); - } - } - - // Try value - if let Some(start) = xml.find(&open_with_attrs) { - let after_tag = &xml[start..]; - let tag_close = after_tag.find('>')?; - let value_start = start + tag_close + 1; - if let Some(end) = xml[value_start..].find(&close) { - return Some(xml[value_start..value_start + end].trim().to_string()); - } - } - - None -} - -/// Extract `value` pairs from EventData section. -/// -/// All values are sanitized to strip control characters (e.g. `\r`, `\0`) that -/// would render as unexpected glyphs in the UI. -fn extract_xml_event_data(xml: &str) -> Vec { - fn data_name_re() -> &'static Regex { - static CELL: OnceLock = OnceLock::new(); - CELL.get_or_init(|| { - Regex::new(r#"(.*?)"#).expect("data regex") - }) - } - - data_name_re() - .captures_iter(xml) - .map(|cap| EvtxField { - name: cap[1].to_string(), - value: sanitize_control_chars(&cap[2]), - }) - .collect() -} - -/// Build a summary message from EventData fields (fallback when EvtFormatMessage unavailable). -fn build_event_data_summary(fields: &[EvtxField]) -> String { - fields - .iter() - .take(5) - .map(|f| { - let val = if f.value.len() > 80 { - format!("{}...", &f.value[..77]) - } else { - f.value.clone() - }; - format!("{}: {val}", f.name) - }) - .collect::>() - .join("; ") -} - // ── Error helpers ─────────────────────────────────────────────────────────── #[cfg(target_os = "windows")] @@ -668,6 +515,9 @@ mod live_service_tests { //! what we send it. use super::*; + // Only the assertions need the level type; the query path itself no longer builds records, so + // importing it at module scope would warn in a non-test build. + use super::super::models::EvtxLevel; use cmtraceopen_parser::event_query::{EventQueryFilter, TimeWindow}; const CHANNEL: &str = "Application"; diff --git a/src-tauri/src/event_log/mod.rs b/src-tauri/src/event_log/mod.rs index 609b7bbd6..f3c9e3dc0 100644 --- a/src-tauri/src/event_log/mod.rs +++ b/src-tauri/src/event_log/mod.rs @@ -5,6 +5,7 @@ pub mod maps; pub mod models; pub mod parser; pub mod provider_db; +pub mod rendered; pub mod timeline; #[cfg(target_os = "windows")] diff --git a/src-tauri/src/event_log/parser.rs b/src-tauri/src/event_log/parser.rs index 6f8e84c01..1e102d7bc 100644 --- a/src-tauri/src/event_log/parser.rs +++ b/src-tauri/src/event_log/parser.rs @@ -4,6 +4,10 @@ use std::sync::RwLock; use cmtraceopen_parser::eventmap::MapRegistry; use evtx::EvtxParser; +// `extract_event_data` sits in `event_node` alongside `extract_system_fields`: both read a parsed +// tree, and both are needed by the live path as well as this one. Keeping the data extractor here +// while the live path scanned raw XML for itself is what let the two drift apart. +use super::event_node::{extract_event_data, EventFields}; use super::provider_db::ProviderStore; use super::models::{ @@ -276,87 +280,6 @@ fn parse_single_file( }) } -/// Extract event fields as name-value pairs. -/// -/// Both `EventData` and `UserData` are read. Manifest providers use the former and classic or -/// trace-backed providers the latter, and skipping `UserData` would leave those events with no -/// fields at all. -/// -/// A `Data` element with no `Name` attribute is numbered by its position, matching how the event -/// message template refers to it. Values are sanitized to strip control characters that would -/// render as unexpected glyphs. -fn extract_event_data(root: &cmtraceopen_parser::eventmap::EventNode) -> EventFields { - let mut fields = Vec::new(); - let mut insertions = Vec::new(); - let mut unnamed = 0usize; - - let containers = root - .children - .iter() - .filter(|child| child.name == "EventData" || child.name == "UserData"); - - // A child that carries text is a field. A child that carries only elements is a wrapper, which - // is how UserData nests its fields under a provider-named element, so it is descended through. - // - // Deciding per child rather than per container matters: an EventData holding only has - // no at all, and treating the whole container as wrappers would descend into , - // find no children, and drop the only value the event carried. - let push_field = |child: &cmtraceopen_parser::eventmap::EventNode, - fields: &mut Vec, - insertions: &mut Vec, - unnamed: &mut usize| { - let value = sanitize_control_chars(child.text.as_deref().unwrap_or_default()); - // Recorded even when empty. The provider's message template addresses fields by position, - // so skipping one here shifts every later %N and renders the description with the wrong - // values substituted into it, which reads as fact. - insertions.push(value.clone()); - - // Counted before the emptiness check, for the same reason. The label is what an operator - // uses to match a field against the provider's template, so skipping the count for a blank - // slot would label the second field Data1 while the template calls it %2. - let position = if child.attribute("Name").is_none() && child.name == "Data" { - *unnamed += 1; - Some(*unnamed) - } else { - None - }; - - if value.is_empty() { - return; - } - let name = match (child.attribute("Name"), position) { - (Some(name), _) => name.to_string(), - (None, Some(position)) => format!("Data{position}"), - (None, None) => child.name.clone(), - }; - fields.push(EvtxField { name, value }); - }; - - for container in containers { - for child in &container.children { - if child.text.is_some() || child.children.is_empty() { - push_field(child, &mut fields, &mut insertions, &mut unnamed); - } else { - for grandchild in &child.children { - push_field(grandchild, &mut fields, &mut insertions, &mut unnamed); - } - } - } - } - - EventFields { fields, insertions } -} - -/// What an event's data section yielded, in the two shapes that are needed. -/// -/// They differ, and conflating them corrupts messages. The display list omits fields the provider -/// left empty, because a column of blanks is noise. The insertion list keeps them, because the -/// message template addresses fields by position and a gap shifts every later reference. -struct EventFields { - fields: Vec, - insertions: Vec, -} - /// Renders the provider's own description for this event, when metadata for it is loaded. /// /// Returns `None` when no database is loaded, the provider is absent from it, or the provider does diff --git a/src-tauri/src/event_log/rendered.rs b/src-tauri/src/event_log/rendered.rs new file mode 100644 index 000000000..7c8e5efb6 --- /dev/null +++ b/src-tauri/src/event_log/rendered.rs @@ -0,0 +1,363 @@ +//! Turning one rendered event into one [`EvtxRecord`]. +//! +//! The live path renders each event to XML with `EvtRender` and then has to translate it. That +//! translation is pure: it takes a string and returns a record, and nothing about it needs the +//! Event Log service. It lives here rather than in `live` so it compiles and is tested on every +//! platform, not only the one that can produce the input. +//! +//! It reads the parsed tree that the map engine already needs, rather than scanning the XML text. +//! The scanning version this replaces re-derived six System fields with substring searches and +//! matched `EventData` with a regular expression, on top of the parse it was doing anyway. That +//! regex required a `Name` attribute, could not match a value containing a newline, and never saw +//! `UserData` at all, so three whole classes of event field were dropped from the live view without +//! anything indicating a field was missing. The file path never had those bugs because it read the +//! tree. Sharing one extractor is what stops the two paths drifting again. + +use cmtraceopen_parser::eventmap::{EventNode, MapRegistry}; + +use super::event_node::{ + extract_event_data, extract_system_fields, parse_event_xml, EventFields, SystemFields, +}; +use super::models::{EvtxField, EvtxLevel, EvtxRecord}; +use super::{parse_timestamp_to_epoch_ms, sanitize_control_chars}; + +/// Names a provider that identified itself in neither of the two ways it can. +/// +/// Matches what the file path shows for the same event. A blank cell reads as a missing column +/// rather than as an event that did not name its source. +const UNKNOWN: &str = "Unknown"; + +/// Builds a record from one rendered event. +/// +/// Returns `None` when the XML will not parse. The caller counts those rather than pushing a record +/// with every field defaulted, which would put a row on screen claiming provider "Unknown" at the +/// epoch and looking exactly like a real event that happened in 1970. +/// +/// The live path calls [`record_from_parts`] instead, because it needs the provider name before the +/// record exists in order to ask the service to render the message. +pub fn parse_xml_to_record( + xml: &str, + channel: &str, + maps: &MapRegistry, + rendered_message: Option<&str>, +) -> Option { + let parsed = parse_event_xml(xml).ok()?; + let system = extract_system_fields(&parsed); + Some(record_from_parts( + &parsed, + system, + xml, + channel, + maps, + rendered_message, + )) +} + +/// Builds a record from a tree and System block the caller has already read. +/// +/// Both are taken as arguments so the document is parsed once per event. The live path needs the +/// provider name first, to name the publisher whose message template it asks the service to render, +/// and parsing again to get the rest would double the cost of the hottest loop in the view. +/// +/// `parsed` is used for the System block, the event fields, the decoded payload and any registered +/// map, so none of them costs an extra pass over the document. +pub fn record_from_parts( + parsed: &EventNode, + system: SystemFields, + xml: &str, + channel: &str, + maps: &MapRegistry, + rendered_message: Option<&str>, +) -> EvtxRecord { + // `Provider` carries `Name` for a manifest provider and `EventSourceName` for a classic one. + // Some carry both, and then `Name` is the identity. `extract_system_fields` already encodes + // that; the live path used to search the raw text for `Name=`, which also matches the tail of + // `EventSourceName=`. + let provider = system.provider.unwrap_or_else(|| UNKNOWN.to_string()); + let event_id = system.event_id.unwrap_or(0); + // Defaulted to the same value the file path uses, so one event cannot show two severities + // depending on how it was opened. `EvtxLevel` maps every unrecognised value to Information, so + // this is Information in practice; it is written as 0 to keep the two paths textually identical + // if that mapping ever gains a distinct "unspecified" variant. + let level = EvtxLevel::from_level_value(system.level.unwrap_or(0)); + let computer = system.computer.unwrap_or_else(|| UNKNOWN.to_string()); + let timestamp = system.time_created.unwrap_or_default(); + let timestamp_epoch = parse_timestamp_to_epoch_ms(×tamp); + let event_record_id = system.event_record_id.unwrap_or(0); + + // The live path renders its message with `EvtFormatMessage`, which does its own substitution + // inside the service, so the positional insertion list has no consumer here. + let EventFields { + fields: mut event_data, + insertions: _, + } = extract_event_data(parsed); + + // Trace-backed channels carry their message as a hex blob rather than as EventData, so without + // this the row reads as a wall of hex digits. Surfaced as a field of its own because the raw + // hex never appears in EventData. Appended after every real field so it cannot disturb the + // positional insertions. + let payload = cmtraceopen_parser::event_payload::decode_payload_in(parsed) + .map(|decoded| sanitize_control_chars(&decoded.text)); + if let Some(text) = &payload { + event_data.push(EvtxField { + name: "EventPayload".to_string(), + value: text.clone(), + }); + } + + // The provider's own rendered message wins when there is one. A decoded payload comes next, + // used whole rather than through the summary, which would truncate the only text the event has + // at 80 characters. Sanitized to strip control characters that render as unexpected glyphs. + let message = rendered_message + .map(sanitize_control_chars) + .or(payload) + .unwrap_or_else(|| build_event_data_summary(&event_data)); + + let mapped = super::maps::apply_registered(maps, channel, &provider, event_id, parsed); + + EvtxRecord { + id: 0, // assigned by commands.rs after sorting + event_record_id, + timestamp, + timestamp_epoch, + provider, + // The queried channel names the record. The caller knows which channel it asked for, and a + // forwarded event names the channel it came from rather than the one holding it. + channel: channel.to_string(), + event_id, + level, + computer, + message, + event_data, + raw_xml: xml.to_string(), + source_label: "Live".to_string(), + task: system.task, + opcode: system.opcode, + process_id: system.process_id, + thread_id: system.thread_id, + user_sid: system.user_sid, + keywords: system.keywords, + mapped, + } +} + +/// Builds a summary from event fields, for events the provider will not describe. +pub fn build_event_data_summary(fields: &[EvtxField]) -> String { + fields + .iter() + .take(5) + .map(|f| { + let val = if f.value.chars().count() > 80 { + // Sliced by character rather than by byte. A value whose 78th byte lands inside a + // multi-byte character panics on a byte slice, and event fields carry paths and + // user names that are routinely not ASCII. + let head: String = f.value.chars().take(77).collect(); + format!("{head}...") + } else { + f.value.clone() + }; + format!("{}: {val}", f.name) + }) + .collect::>() + .join("; ") +} + +#[cfg(test)] +mod tests { + //! `EvtRender` writes attributes in single quotes, so the fixtures do too. Double-quoted + //! fixtures would exercise a shape the service never emits. + + use super::*; + + fn record_for(xml: &str, channel: &str) -> EvtxRecord { + parse_xml_to_record(xml, channel, &MapRegistry::new(), None) + .expect("a well formed event should produce a record") + } + + fn field<'a>(record: &'a EvtxRecord, name: &str) -> Option<&'a str> { + record + .event_data + .iter() + .find(|f| f.name == name) + .map(|f| f.value.as_str()) + } + + /// Manifest provider: `Name` is present, and that is the identity to use. + const MANIFEST: &str = r#" + + + 4624 + 0 + + 4242 + Security + RING0IVY24-01 + + + 0x8020000000000000 + + + adam + +"#; + + #[test] + fn a_manifest_provider_is_read_from_name() { + let record = record_for(MANIFEST, "Security"); + assert_eq!(record.provider, "Microsoft-Windows-Security-Auditing"); + assert_eq!(record.event_id, 4624); + assert_eq!(record.event_record_id, 4242); + assert_eq!(record.computer, "RING0IVY24-01"); + assert_eq!(record.process_id, Some(1234)); + assert_eq!(record.thread_id, Some(5678)); + assert_eq!(record.user_sid.as_deref(), Some("S-1-5-18")); + assert_eq!(record.keywords.as_deref(), Some("0x8020000000000000")); + assert_eq!(field(&record, "TargetUserName"), Some("adam")); + } + + #[test] + fn a_classic_source_still_names_its_provider() { + // A classic source names itself only in EventSourceName. Losing it leaves the row blank and + // stops every map from matching, and neither failure looks like a failure. + let xml = r#" + + + 10 + 4 + Application + +"#; + let record = record_for(xml, "Application"); + assert_eq!(record.provider, "Print"); + assert_eq!(record.event_id, 10, "Qualifiers is not part of the id"); + } + + #[test] + fn a_manifest_name_wins_over_a_legacy_source_alias() { + // Both attributes are present, and the text `Name='` occurs inside `EventSourceName='`. + // Anything searching the raw XML for that substring can match the alias and report the + // wrong provider depending only on attribute order. + let xml = r#" + + + 16384 + +"#; + assert_eq!( + record_for(xml, "Application").provider, + "Microsoft-Windows-Security-SPP" + ); + } + + #[test] + fn an_unnamed_data_element_is_kept_and_numbered_by_position() { + // Classic providers write positional Data with no Name. Requiring a Name attribute dropped + // every one of them, so the event arrived with no fields and the detail pane was empty. + let xml = r#" + 1033 + + Product Name + 1.2.3 + +"#; + let record = record_for(xml, "Application"); + assert_eq!(field(&record, "Data1"), Some("Product Name")); + assert_eq!(field(&record, "Data2"), Some("1.2.3")); + } + + #[test] + fn a_multi_line_value_survives() { + // Stack traces, path lists and MSI output all span lines. A regex whose `.` stops at a + // newline matched none of them, and the field vanished rather than arriving truncated, + // which is the harder failure to notice. + let xml = "\n 1000\n \n at Frame.One\nat Frame.Two\n \n"; + assert_eq!( + field(&record_for(xml, "Application"), "Trace"), + Some("at Frame.One\nat Frame.Two") + ); + } + + #[test] + fn user_data_fields_are_read() { + // Trace-backed and classic providers put their fields in UserData, nested under a + // provider-named wrapper. Reading only EventData left these events with no fields at all. + let xml = r#" + 811 + + + WinlogonSubscription + + +"#; + assert_eq!( + field(&record_for(xml, "Application"), "SubscriptionId"), + Some("WinlogonSubscription") + ); + } + + #[test] + fn an_absent_level_defaults_to_what_the_file_path_shows() { + // The two paths must default identically, or the same event shows one severity when opened + // from a file and another when read live. They previously wrote different literals, 0 here + // and 4 there, which happens to be the same value only because `EvtxLevel` folds every + // unrecognised level into Information. + let xml = r#" + 1 +"#; + assert_eq!( + record_for(xml, "Application").level, + EvtxLevel::from_level_value(0) + ); + } + + #[test] + fn an_unnamed_provider_is_reported_as_unknown_not_as_blank() { + let xml = r#" + 1 +"#; + assert_eq!(record_for(xml, "Application").provider, UNKNOWN); + } + + #[test] + fn malformed_xml_yields_no_record_rather_than_an_empty_one() { + // A record with every field defaulted is indistinguishable on screen from a real event that + // happened at the epoch with no provider. + assert!( + parse_xml_to_record("", "Application", &MapRegistry::new(), None) + .is_none() + ); + } + + #[test] + fn the_rendered_message_is_preferred_over_a_field_summary() { + let record = parse_xml_to_record( + MANIFEST, + "Security", + &MapRegistry::new(), + Some("An account was successfully logged on."), + ) + .expect("record"); + assert_eq!(record.message, "An account was successfully logged on."); + } + + #[test] + fn without_a_rendered_message_the_fields_are_summarised() { + assert_eq!( + record_for(MANIFEST, "Security").message, + "TargetUserName: adam" + ); + } + + #[test] + fn a_long_non_ascii_value_is_truncated_without_panicking() { + // Truncating by byte offset panics when the cut lands inside a multi-byte character, and + // event fields carry paths and account names that are routinely not ASCII. + let value = "é".repeat(200); + let summary = build_event_data_summary(&[EvtxField { + name: "Path".to_string(), + value, + }]); + assert!(summary.ends_with("...")); + assert_eq!(summary.chars().filter(|c| *c == 'é').count(), 77); + } +} From 05ae28380603c8830105323c9c6f54a4285d5842 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 11 Aug 2026 22:42:54 -0400 Subject: [PATCH 72/85] test(event-log): add a timing harness for the live scan Phase 1 of the event viewer epic carries a gate: no performance claim ships without a reproducible scenario and recorded numbers. There was no way to produce either. This adds the scan half - channel enumeration and a windowed query across every channel, with wall clock and per-event cost - so a claim can be checked rather than argued from reading the code. Channels that fail to read are counted rather than skipped, because treating an unreadable channel as zero events reports a faster scan of a smaller corpus as an improvement. Peak working set is deliberately left to the caller: a process cannot sample its own peak reliably. Co-Authored-By: Claude Opus 5 --- src-tauri/Cargo.toml | 6 ++ src-tauri/examples/evtx_scan.rs | 109 ++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 src-tauri/examples/evtx_scan.rs diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index ecd4e045d..4ef6c64dc 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -136,6 +136,12 @@ tempfile = "3" plist = "1" tauri = { version = "2", features = ["test"] } +# Timing harness for the live event-log scan. Declared so `--all-targets` does not try to build it +# without the modules it measures. +[[example]] +name = "evtx_scan" +required-features = ["event-log"] + [[test]] name = "esp_diagnostics_sources" required-features = ["esp-diagnostics"] diff --git a/src-tauri/examples/evtx_scan.rs b/src-tauri/examples/evtx_scan.rs new file mode 100644 index 000000000..37450e1c1 --- /dev/null +++ b/src-tauri/examples/evtx_scan.rs @@ -0,0 +1,109 @@ +//! Times a live scan so a performance claim can be checked instead of asserted. +//! +//! This exists because the epic's Phase 1 gate requires a reproducible scenario with recorded +//! numbers, and because every change in the live path so far has been argued from reading the code +//! rather than from measurement. Wall clock is printed here; peak working set is measured by the +//! caller, since a process cannot observe its own peak as reliably as the parent can. +//! +//! Windows only, because there is no Event Log service to scan anywhere else. On other platforms it +//! prints why it did nothing rather than reporting a zero that would look like a fast scan. +//! +//! ```text +//! cargo run --release --example evtx_scan --features event-log -- --days 7 +//! cargo run --release --example evtx_scan --features event-log -- --days 7 --channel Application +//! ``` + +fn main() { + let args: Vec = std::env::args().collect(); + let value_of = |flag: &str| -> Option { + args.iter() + .position(|a| a == flag) + .and_then(|i| args.get(i + 1)) + .cloned() + }; + + let days: u64 = value_of("--days") + .and_then(|v| v.parse().ok()) + .unwrap_or(7); + let only_channel = value_of("--channel"); + // Absent means no cap, which is the case the gate cares about: the cap is the thing being + // replaced, so measuring with it in place would measure the cap rather than the scan. + let max_events: Option = value_of("--max").and_then(|v| v.parse().ok()); + + run(days, only_channel, max_events); +} + +#[cfg(target_os = "windows")] +fn run(days: u64, only_channel: Option, max_events: Option) { + use app_lib::event_log::live; + use cmtraceopen_parser::event_query::{EventQueryFilter, TimeWindow}; + use cmtraceopen_parser::eventmap::MapRegistry; + use std::time::Instant; + + let enumerated = Instant::now(); + let channels = match live::enumerate_channels() { + Ok(channels) => channels, + Err(error) => { + eprintln!("enumerate_channels failed: {error}"); + std::process::exit(1); + } + }; + let channels: Vec = channels + .into_iter() + .map(|c| c.name) + .filter(|name| only_channel.as_ref().is_none_or(|only| only == name)) + .collect(); + let enumerate_ms = enumerated.elapsed().as_millis(); + + let filter = EventQueryFilter { + time: Some(TimeWindow::Last { + milliseconds: days * 24 * 60 * 60 * 1000, + }), + ..Default::default() + }; + + // No maps loaded. The map engine has its own benchmark; mixing it in here would make a change + // to either one move this number. + let maps = MapRegistry::new(); + + let mut total = 0usize; + let mut failed = 0usize; + let mut slowest = (String::new(), 0u128); + + let started = Instant::now(); + for channel in &channels { + let at = Instant::now(); + match live::query_channel_filtered(channel, &filter, &maps, max_events) { + Ok(records) => total += records.len(), + // A channel that cannot be read is counted, not ignored. Treating it as zero events + // would report a faster scan of a smaller corpus as an improvement. + Err(_) => failed += 1, + } + let took = at.elapsed().as_millis(); + if took > slowest.1 { + slowest = (channel.clone(), took); + } + } + let elapsed = started.elapsed(); + + let per_event_us = if total > 0 { + elapsed.as_micros() as f64 / total as f64 + } else { + 0.0 + }; + + println!("days={days}"); + println!("channels_scanned={}", channels.len()); + println!("channels_failed={failed}"); + println!("events={total}"); + println!("enumerate_ms={enumerate_ms}"); + println!("scan_ms={}", elapsed.as_millis()); + println!("per_event_us={per_event_us:.2}"); + println!("slowest_channel={} ({}ms)", slowest.0, slowest.1); +} + +#[cfg(not(target_os = "windows"))] +fn run(_days: u64, _only_channel: Option, _max_events: Option) { + eprintln!("evtx_scan needs a Windows Event Log service; nothing to measure on this platform."); + std::process::exit(2); +} From 38f66bedfdefc505500a8eeddeb4319ede5a1c72 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 11 Aug 2026 22:47:59 -0400 Subject: [PATCH 73/85] fix(event-log): query channels one at a time, not all in one request queryChannels sent every selected channel in a single request. The backend collects a whole request's records into one vector before replying, so asking for forty channels held every event of every channel in memory, twice, before anything reached the screen. The two other load paths in this store already query per channel; this one was the outlier, and it is the path that loads the channels a user explicitly selects. A single request also fails as a whole, so one unreadable channel discarded the results of every channel queried alongside it and left the view empty. Each channel now succeeds or fails on its own, and a channel that could not be read is recorded as a coverage gap rather than only as a loadError string that the next load replaces. Co-Authored-By: Claude Opus 5 --- .../event-log/evtx-store-coverage.test.ts | 91 +++++++++++++++++++ src/workspaces/event-log/evtx-store.ts | 61 +++++++++---- 2 files changed, 136 insertions(+), 16 deletions(-) diff --git a/src/workspaces/event-log/evtx-store-coverage.test.ts b/src/workspaces/event-log/evtx-store-coverage.test.ts index 339b902ec..37ff8d5fd 100644 --- a/src/workspaces/event-log/evtx-store-coverage.test.ts +++ b/src/workspaces/event-log/evtx-store-coverage.test.ts @@ -92,6 +92,97 @@ describe("coverage gaps through the store", () => { }); }); +describe("a multi-channel query is delivered one channel at a time", () => { + beforeEach(() => { + invoke.mockReset(); + useEvtxStore.setState({ + records: [], + channels: [], + coverageGaps: [], + loadedChannels: new Set(), + selectedChannels: new Set(), + }); + }); + + function withRecords(channel: string, count: number) { + return { + records: Array.from({ length: count }, (_, i) => ({ + id: i, + eventRecordId: i, + timestamp: "2026-08-11T12:00:00.000Z", + timestampEpoch: 1_000 + i, + provider: "P", + channel, + eventId: 1, + level: "information", + computer: "C", + message: "m", + eventData: [], + rawXml: "", + sourceLabel: "Live", + mapped: null, + })), + channels: [{ name: channel, eventCount: count, sourceType: "live" }], + totalRecords: count, + parseErrors: 0, + errorMessages: [], + }; + } + + it("asks for each channel separately rather than all of them at once", async () => { + // The backend collects a whole request into one vector before replying, so a single request + // naming every channel holds every event of every channel in memory before anything is shown. + invoke + .mockResolvedValueOnce(withRecords("Application", 1)) + .mockResolvedValueOnce(withRecords("System", 1)); + + await useEvtxStore.getState().queryChannels(["Application", "System"]); + + expect(invoke).toHaveBeenCalledTimes(2); + const requested = invoke.mock.calls.map( + (call) => (call[1] as { channels: string[] }).channels + ); + expect(requested).toEqual([["Application"], ["System"]]); + }); + + it("keeps the channels that succeeded when one of them fails", async () => { + // A single request fails as a whole. One unreadable channel used to discard the results of + // every channel queried alongside it and leave the view empty. + invoke + .mockRejectedValueOnce(new Error("access denied")) + .mockResolvedValueOnce(withRecords("System", 3)); + + await useEvtxStore.getState().queryChannels(["Security", "System"]); + + const state = useEvtxStore.getState(); + expect(state.records).toHaveLength(3); + expect(state.loadedChannels.has("System")).toBe(true); + expect(state.loadedChannels.has("Security")).toBe(false); + }); + + it("records the unreadable channel as a gap, not only as an error string", async () => { + // loadError is replaced by the next load. A gap describes the view currently on screen, and + // events that never arrived look exactly like evidence that nothing happened. + invoke + .mockRejectedValueOnce(new Error("access denied")) + .mockResolvedValueOnce(withRecords("System", 1)); + + await useEvtxStore.getState().queryChannels(["Security", "System"]); + + const gaps = useEvtxStore.getState().coverageGaps; + expect(gaps.some((g) => g.includes("Security") && g.includes("access denied"))).toBe(true); + }); + + it("still reports the failure through loadError", async () => { + invoke.mockRejectedValueOnce(new Error("access denied")); + + await useEvtxStore.getState().queryChannels(["Security"]); + + expect(useEvtxStore.getState().loadError).toContain("Security"); + expect(useEvtxStore.getState().isLoading).toBe(false); + }); +}); + describe("the time window reaches the service", () => { beforeEach(() => { invoke.mockReset(); diff --git a/src/workspaces/event-log/evtx-store.ts b/src/workspaces/event-log/evtx-store.ts index e8ac0cb41..29d00bae2 100644 --- a/src/workspaces/event-log/evtx-store.ts +++ b/src/workspaces/event-log/evtx-store.ts @@ -252,16 +252,49 @@ export const useEvtxStore = create()((set, get) => ({ }, queryChannels: async (channels, maxEvents) => { - set({ isLoading: true, loadError: null }); - try { - const result = await invoke("evtx_query_channels", { - channels, - maxEvents: maxEvents ?? null, - filter: buildServerFilter(get().timeWindow), - }); - const checked = assertParseResultShape(result); + set({ isLoading: true, loadError: null, selectedRecordId: null }); + + // One request per channel rather than one request for all of them. The backend collects a + // whole request's records into a single vector before replying, so asking for forty channels + // at once held every event of every channel in memory twice, once per channel and once in the + // combined vector, before anything reached the screen. + // + // It also isolates failure. A single request fails as a whole, so one unreadable channel threw + // away the results of every channel queried alongside it and left the view empty. + let loadError: string | null = null; + + const results = await Promise.all( + channels.map(async (ch) => { + try { + const result = await invoke("evtx_query_channels", { + channels: [ch], + maxEvents: maxEvents ?? null, + filter: buildServerFilter(get().timeWindow), + }); + return { channel: ch, result, error: null as string | null }; + } catch (e) { + const message = e instanceof Error ? e.message : String(e); + console.warn(`[evtx] Failed to query ${ch}: ${message}`); + if (!loadError) loadError = `${ch}: ${message}`; + return { channel: ch, result: null, error: message }; + } + }) + ); - // Merge new records with existing ones (for incremental channel loading) + for (const { channel, result, error } of results) { + if (!result) { + // A channel that could not be read is recorded as a gap, not merely as an error banner + // that the next successful load replaces. The events it would have contributed are absent + // from the view for as long as the view is on screen. + set((s) => ({ + coverageGaps: mergeCoverageGaps(s.coverageGaps, [ + `${channel}: not read (${error ?? "unknown error"})`, + ]), + })); + continue; + } + + const checked = assertParseResultShape(result); const state = get(); const existingChannelNames = new Set(state.records.map((r) => r.channel)); // Only add records from channels we don't already have @@ -279,23 +312,19 @@ export const useEvtxStore = create()((set, get) => ({ })); const newLoaded = new Set(state.loadedChannels); - for (const ch of channels) newLoaded.add(ch); + newLoaded.add(channel); set({ records: merged, channels: updatedChannels, loadedChannels: newLoaded, - isLoading: false, - loadError: null, // Accumulated, not dropped. This path loads channels incrementally, so discarding what the // backend reported here would show a complete view of a partly unreadable set. coverageGaps: mergeCoverageGaps(state.coverageGaps, checked.errorMessages), - selectedRecordId: null, }); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - set({ isLoading: false, loadError: message }); } + + set({ isLoading: false, loadError }); }, loadSelectedChannels: async () => { From 8964b2d7db2826b5049be166342af8027db609ab Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 11 Aug 2026 22:53:36 -0400 Subject: [PATCH 74/85] docs(event-log): stop claiming a default event cap that does not exist query_channel documented "capped at max_events (default 1000)". There is no default: None becomes usize::MAX, and every caller in the application passes None. A bound that is documented but not enforced is worse than either having it or not, because it invites a caller to rely on it. Co-Authored-By: Claude Opus 5 --- src-tauri/src/event_log/live.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src-tauri/src/event_log/live.rs b/src-tauri/src/event_log/live.rs index 380c97352..af3cea0c6 100644 --- a/src-tauri/src/event_log/live.rs +++ b/src-tauri/src/event_log/live.rs @@ -121,7 +121,10 @@ pub fn enumerate_channels() -> Result, String> { /// Query events from a live Windows Event Log channel. /// -/// Returns newest events first, capped at `max_events` (default 1000). +/// Returns newest events first. `None` means no cap, which is what every caller in the application +/// passes; there is no default limit, and the comment claiming a default of 1000 described a +/// behaviour this function has not had. A cap that is documented but absent is worse than either, +/// because it invites callers to rely on a bound nothing enforces. #[cfg(target_os = "windows")] pub fn query_channel( channel: &str, From 5086b7e3938729e8194d53c3bf2cb081cc97b78a Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 11 Aug 2026 23:22:34 -0400 Subject: [PATCH 75/85] test(event-log): account for where a scan's memory goes The first run of the harness showed the working set climbing past 690MB on an all-channels seven-day scan, which is the constraint that actually matters here and was not being attributed to anything. Every record carries the whole rendered XML it was built from, and that string is serialized to the frontend and held there too, so if it dominates the record it dominates three copies. Reports raw_xml bytes against message and field bytes so that claim is measured rather than assumed, plus the widest channel, since a single channel deciding the peak is a different problem from the total being large. Co-Authored-By: Claude Opus 5 --- src-tauri/examples/evtx_scan.rs | 39 +++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/src-tauri/examples/evtx_scan.rs b/src-tauri/examples/evtx_scan.rs index 37450e1c1..84fce80af 100644 --- a/src-tauri/examples/evtx_scan.rs +++ b/src-tauri/examples/evtx_scan.rs @@ -22,9 +22,7 @@ fn main() { .cloned() }; - let days: u64 = value_of("--days") - .and_then(|v| v.parse().ok()) - .unwrap_or(7); + let days: u64 = value_of("--days").and_then(|v| v.parse().ok()).unwrap_or(7); let only_channel = value_of("--channel"); // Absent means no cap, which is the case the gate cares about: the cap is the thing being // replaced, so measuring with it in place would measure the cap rather than the scan. @@ -69,12 +67,32 @@ fn run(days: u64, only_channel: Option, max_events: Option) { let mut total = 0usize; let mut failed = 0usize; let mut slowest = (String::new(), 0u128); + // Attributing memory rather than guessing at it. Every record carries the whole rendered XML it + // was built from, and that string is then serialized to the frontend and held there too, so if + // it dominates the record it dominates three copies rather than one. + let mut xml_bytes = 0usize; + let mut field_bytes = 0usize; + let mut widest_channel = (String::new(), 0usize); let started = Instant::now(); for channel in &channels { let at = Instant::now(); match live::query_channel_filtered(channel, &filter, &maps, max_events) { - Ok(records) => total += records.len(), + Ok(records) => { + total += records.len(); + if records.len() > widest_channel.1 { + widest_channel = (channel.clone(), records.len()); + } + for record in &records { + xml_bytes += record.raw_xml.len(); + field_bytes += record.message.len() + + record + .event_data + .iter() + .map(|f| f.name.len() + f.value.len()) + .sum::(); + } + } // A channel that cannot be read is counted, not ignored. Treating it as zero events // would report a faster scan of a smaller corpus as an improvement. Err(_) => failed += 1, @@ -100,6 +118,19 @@ fn run(days: u64, only_channel: Option, max_events: Option) { println!("scan_ms={}", elapsed.as_millis()); println!("per_event_us={per_event_us:.2}"); println!("slowest_channel={} ({}ms)", slowest.0, slowest.1); + println!( + "widest_channel={} ({} events)", + widest_channel.0, widest_channel.1 + ); + println!("raw_xml_mb={:.1}", xml_bytes as f64 / (1024.0 * 1024.0)); + println!( + "message_and_fields_mb={:.1}", + field_bytes as f64 / (1024.0 * 1024.0) + ); + if xml_bytes + field_bytes > 0 { + let share = xml_bytes as f64 / (xml_bytes + field_bytes) as f64 * 100.0; + println!("raw_xml_share_pct={share:.0}"); + } } #[cfg(not(target_os = "windows"))] From fd6fdbf6f13423b852e727e49d66a6cfa7eb6804 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 11 Aug 2026 23:35:00 -0400 Subject: [PATCH 76/85] fix(event-log): stop reporting a truncated channel as a complete one Measuring a full seven-day scan surfaced EvtNext failing with RPC_S_INVALID_BOUND (0x800706c6) on one channel out of roughly twelve hundred. The service was refusing the size of the request, not its contents, and the loop's response was to stop reading and return what it already had. The caller received Ok, counted the events, and showed the channel as fully loaded. This is the failure this view exists to avoid: events that were never fetched look exactly like events that do not exist. It was reachable only because the EvtNext batch was raised from 16 to 256 without the measurement the epic asked for, and the larger request is what some channels reject. The batch now halves down to a floor of 8 and retries, which reads the channel rather than abandoning it. If a read still fails, the records already gathered are returned, and the reason is carried alongside them: query_channel_inner and its four wrappers return ChannelScan, a set of records plus the gaps in it, so a caller cannot take the records without also being handed what is missing. Unparsable events are reported the same way instead of only being logged. Co-Authored-By: Claude Opus 5 --- src-tauri/examples/evtx_scan.rs | 9 +- src-tauri/src/event_log/commands.rs | 60 +++++++------ src-tauri/src/event_log/live.rs | 129 +++++++++++++++++++++------- 3 files changed, 141 insertions(+), 57 deletions(-) diff --git a/src-tauri/examples/evtx_scan.rs b/src-tauri/examples/evtx_scan.rs index 84fce80af..cbdbe3c24 100644 --- a/src-tauri/examples/evtx_scan.rs +++ b/src-tauri/examples/evtx_scan.rs @@ -73,12 +73,18 @@ fn run(days: u64, only_channel: Option, max_events: Option) { let mut xml_bytes = 0usize; let mut field_bytes = 0usize; let mut widest_channel = (String::new(), 0usize); + // A channel read only partly is neither a success nor a failure, and counting it as + // either hides it. The scan's own truncations are the thing most likely to make a + // faster number look like an improvement. + let mut gap_reports = 0usize; let started = Instant::now(); for channel in &channels { let at = Instant::now(); match live::query_channel_filtered(channel, &filter, &maps, max_events) { - Ok(records) => { + Ok(scan) => { + let records = scan.records; + gap_reports += scan.gaps.len(); total += records.len(); if records.len() > widest_channel.1 { widest_channel = (channel.clone(), records.len()); @@ -113,6 +119,7 @@ fn run(days: u64, only_channel: Option, max_events: Option) { println!("days={days}"); println!("channels_scanned={}", channels.len()); println!("channels_failed={failed}"); + println!("channels_with_gaps={gap_reports}"); println!("events={total}"); println!("enumerate_ms={enumerate_ms}"); println!("scan_ms={}", elapsed.as_millis()); diff --git a/src-tauri/src/event_log/commands.rs b/src-tauri/src/event_log/commands.rs index c5bafea8b..89227bb7d 100644 --- a/src-tauri/src/event_log/commands.rs +++ b/src-tauri/src/event_log/commands.rs @@ -73,30 +73,29 @@ pub async fn evtx_query_channels( // Event Log service that spends nearly all its time waiting on RPC, so serializing // them leaves the machine idle. Results are collected per channel and ordered // afterwards, so concurrency cannot affect the output. - let per_channel: Vec<(String, Result, String>)> = - channels - .par_iter() - .map(|channel| { - let app_ref = &app; - let ch_name = channel.clone(); - let outcome = super::live::query_channel_filtered_with_progress( - channel, - &query_filter, - &maps, - max_events, - |fetched, _| { - let _ = app_ref.emit( - "evtx-query-progress", - EvtxQueryProgress { - channel: ch_name.clone(), - fetched, - }, - ); - }, - ); - (channel.clone(), outcome) - }) - .collect(); + let per_channel: Vec<(String, Result)> = channels + .par_iter() + .map(|channel| { + let app_ref = &app; + let ch_name = channel.clone(); + let outcome = super::live::query_channel_filtered_with_progress( + channel, + &query_filter, + &maps, + max_events, + |fetched, _| { + let _ = app_ref.emit( + "evtx-query-progress", + EvtxQueryProgress { + channel: ch_name.clone(), + fetched, + }, + ); + }, + ); + (channel.clone(), outcome) + }) + .collect(); let mut all_records = Vec::new(); let mut channel_infos = Vec::new(); @@ -105,13 +104,20 @@ pub async fn evtx_query_channels( for (channel, outcome) in per_channel { match outcome { - Ok(records) => { + Ok(scan) => { channel_infos.push(super::models::EvtxChannelInfo { name: channel.clone(), - event_count: records.len() as u64, + event_count: scan.records.len() as u64, source_type: super::models::ChannelSourceType::Live, }); - all_records.extend(records); + // A channel can be read partly. Those gaps travel with the records so a + // truncated channel is not presented as a complete one, which is what + // happened while the query returned records alone. + if !scan.gaps.is_empty() { + parse_errors += scan.gaps.len() as u32; + error_messages.extend(scan.gaps); + } + all_records.extend(scan.records); } Err(e) => { log::warn!( diff --git a/src-tauri/src/event_log/live.rs b/src-tauri/src/event_log/live.rs index af3cea0c6..b52e1775b 100644 --- a/src-tauri/src/event_log/live.rs +++ b/src-tauri/src/event_log/live.rs @@ -23,6 +23,26 @@ use windows::Win32::System::EventLog::{ #[cfg(target_os = "windows")] const EVENT_FETCH_BATCH: usize = 256; +/// Smallest batch to fall back to before treating the channel as unreadable. +/// +/// Some channels reject a 256-handle request with `RPC_S_INVALID_BOUND`. Measuring a full scan +/// found one doing exactly that, and the loop's response was to stop reading and return what it +/// already had as a complete result. Halving down to this floor reads the channel instead. +#[cfg(target_os = "windows")] +const MIN_FETCH_BATCH: usize = 8; + +/// What one channel yielded, including why anything is missing from it. +/// +/// The records used to be returned on their own, which left no way to say "this channel was read, +/// but not all of it". A partial read then reported as a complete one: the caller saw `Ok`, counted +/// the events, and showed a channel that looked fully loaded. Events that were never fetched are +/// indistinguishable on screen from events that do not exist. +pub struct ChannelScan { + pub records: Vec, + /// Operator-facing explanations of what is missing. Empty means the channel was read whole. + pub gaps: Vec, +} + // ── RAII handle wrapper ───────────────────────────────────────────────────── #[cfg(target_os = "windows")] @@ -130,7 +150,7 @@ pub fn query_channel( channel: &str, maps: &MapRegistry, max_events: Option, -) -> Result, String> { +) -> Result { query_channel_with_progress(channel, maps, max_events, |_, _| {}) } @@ -144,7 +164,7 @@ pub fn query_channel_filtered( filter: &EventQueryFilter, maps: &MapRegistry, max_events: Option, -) -> Result, String> { +) -> Result { query_channel_inner(channel, filter, maps, max_events, |_, _| {}) } @@ -155,7 +175,7 @@ pub fn query_channel_with_progress( maps: &MapRegistry, max_events: Option, on_progress: impl Fn(usize, Option), -) -> Result, String> { +) -> Result { query_channel_inner( channel, &EventQueryFilter::default(), @@ -173,7 +193,7 @@ pub fn query_channel_filtered_with_progress( maps: &MapRegistry, max_events: Option, on_progress: impl Fn(usize, Option), -) -> Result, String> { +) -> Result { query_channel_inner(channel, filter, maps, max_events, on_progress) } @@ -184,7 +204,7 @@ fn query_channel_inner( maps: &MapRegistry, max_events: Option, on_progress: impl Fn(usize, Option), -) -> Result, String> { +) -> Result { let limit = max_events.map(|n| n as usize).unwrap_or(usize::MAX); let channel_hstring = HSTRING::from(channel); // A filter that cannot be expressed is refused here rather than silently degraded to "*", @@ -212,21 +232,51 @@ fn query_channel_inner( let mut publisher_metadata = HashMap::>::new(); let mut unparsable = 0usize; + let mut gaps = Vec::new(); + let mut batch = EVENT_FETCH_BATCH; + while records.len() < limit { let mut raw_handles = [0isize; EVENT_FETCH_BATCH]; let mut returned = 0u32; - match unsafe { EvtNext(query_handle.raw(), &mut raw_handles, 0, 0, &mut returned) } { + match unsafe { + EvtNext( + query_handle.raw(), + &mut raw_handles[..batch], + 0, + 0, + &mut returned, + ) + } { Ok(()) => {} + Err(e) if is_no_more_items(&e) => break, + // The service can refuse a batch this large on a particular channel. Halving and + // retrying reads it; the previous behaviour was to stop and report what had already + // been read as the channel's full contents. + Err(e) if is_invalid_bound(&e) && batch > MIN_FETCH_BATCH => { + batch = (batch / 2).max(MIN_FETCH_BATCH); + log::info!( + "event=evtx_batch_reduced channel=\"{channel}\" batch={batch} \ + reason=\"the service rejected the previous batch size\"" + ); + continue; + } Err(e) => { - if !is_no_more_items(&e) { - eprintln!( - "[evtx] EvtNext error: code=0x{:08x} w32={} msg=\"{}\"", - e.code().0 as u32, - win32_code(&e), - e.message() - ); - } + // Recorded as a gap, not just logged. The records already read are still returned, + // because they are real, but the channel must not be presented as complete. + let detail = format!( + "{channel}: stopped after {} events, the channel could not be read further ({}, 0x{:08x})", + records.len(), + e.message().trim(), + e.code().0 as u32 + ); + log::warn!( + "event=evtx_next_failed channel=\"{channel}\" batch={batch} \ + w32={} code=0x{:08x}", + win32_code(&e), + e.code().0 as u32 + ); + gaps.push(detail); break; } } @@ -295,12 +345,16 @@ fn query_channel_inner( // Counted and reported rather than passed over. Events that never arrived look exactly like // evidence that the thing being investigated did not happen. log::warn!("event=evtx_live_query_gap channel=\"{channel}\" unparsable={unparsable}"); + gaps.push(format!( + "{channel}: {unparsable} events could not be read and are missing from this view" + )); } log::info!( - "event=evtx_live_query_done channel=\"{channel}\" records={} unparsable={unparsable}", - records.len() + "event=evtx_live_query_done channel=\"{channel}\" records={} unparsable={unparsable} gaps={}", + records.len(), + gaps.len() ); - Ok(records) + Ok(ChannelScan { records, gaps }) } // ── Non-Windows stubs ─────────────────────────────────────────────────────── @@ -316,7 +370,7 @@ pub fn query_channel_with_progress( _maps: &MapRegistry, _max_events: Option, _on_progress: impl Fn(usize, Option), -) -> Result, String> { +) -> Result { Err("Live event log queries are only available on Windows.".to_string()) } @@ -325,7 +379,7 @@ pub fn query_channel( _channel: &str, _maps: &MapRegistry, _max_events: Option, -) -> Result, String> { +) -> Result { Err("Live event log queries are only available on Windows.".to_string()) } @@ -335,7 +389,7 @@ pub fn query_channel_filtered( _filter: &EventQueryFilter, _maps: &MapRegistry, _max_events: Option, -) -> Result, String> { +) -> Result { Err("Live event log queries are only available on Windows.".to_string()) } @@ -346,7 +400,7 @@ pub fn query_channel_filtered_with_progress( _maps: &MapRegistry, _max_events: Option, _on_progress: impl Fn(usize, Option), -) -> Result, String> { +) -> Result { Err("Live event log queries are only available on Windows.".to_string()) } @@ -464,6 +518,16 @@ fn is_no_more_items(error: &Error) -> bool { win32_code(error) == 259 } +/// `RPC_S_INVALID_BOUND`: the service refused the size of the request, not its contents. +/// +/// Observed from `EvtNext` with a 256-handle batch on a real machine, on one channel out of roughly +/// twelve hundred. It says nothing about the channel's data, so the right response is a smaller +/// request rather than abandoning the channel. +#[cfg(target_os = "windows")] +fn is_invalid_bound(error: &Error) -> bool { + win32_code(error) == 1734 +} + #[cfg(target_os = "windows")] fn is_not_found(error: &Error) -> bool { win32_code(error) == 1168 @@ -486,8 +550,9 @@ mod tests { let has_app = channels.iter().any(|c| c.name == "Application"); println!("Has Application channel: {has_app}"); - let records = - query_channel("Application", &MapRegistry::new(), Some(3)).expect("query should work"); + let records = query_channel("Application", &MapRegistry::new(), Some(3)) + .expect("query should work") + .records; println!("Application records: {}", records.len()); for (i, r) in records.iter().enumerate() { println!("--- Record {i} ---"); @@ -534,7 +599,8 @@ mod live_service_tests { #[test] #[ignore = "requires a live Windows Event Log service with events"] fn an_unfiltered_query_returns_records() { - let records = query_channel(CHANNEL, &no_maps(), Some(50)).expect("query succeeds"); + let scan = query_channel(CHANNEL, &no_maps(), Some(50)).expect("query succeeds"); + let records = scan.records; assert!( !records.is_empty(), "Application channel should have events" @@ -564,7 +630,8 @@ mod live_service_tests { &no_maps(), None, ) - .expect("1 hour query succeeds"); + .expect("1 hour query succeeds") + .records; let wide = query_channel_filtered( CHANNEL, @@ -577,7 +644,8 @@ mod live_service_tests { &no_maps(), None, ) - .expect("30 day query succeeds"); + .expect("30 day query succeeds") + .records; assert!( narrow.len() <= wide.len(), @@ -601,7 +669,8 @@ mod live_service_tests { &no_maps(), Some(200), ) - .expect("level query succeeds"); + .expect("level query succeeds") + .records; for record in &records { assert_eq!( @@ -628,7 +697,8 @@ mod live_service_tests { &no_maps(), Some(50), ) - .expect("query succeeds"); + .expect("query succeeds") + .records; assert!( records.is_empty(), @@ -642,7 +712,8 @@ mod live_service_tests { fn system_fields_are_populated_from_real_events() { let records = query_channel_filtered(CHANNEL, &EventQueryFilter::default(), &no_maps(), Some(200)) - .expect("query succeeds"); + .expect("query succeeds") + .records; assert!(!records.is_empty()); assert!( From 3c9cca1432e12b174947bebc8ec790d10bae2ef2 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 11 Aug 2026 23:42:46 -0400 Subject: [PATCH 77/85] refactor(event-log): move the fetch-failure decision where CI can test it The rule for responding to a failed EvtNext - end of channel, retry smaller, or report truncation - was written inside the Windows-only query loop, so no CI runner could execute it. That is the wrong place for it. Getting it wrong does not crash: it returns a partly read channel that the caller presents as whole, which is the failure mode this view exists to avoid, and it is exactly the kind of quiet wrong answer that needs a test running everywhere. classify_fetch_failure is a pure function of the Win32 code, the current batch and the floor, covered by six tests including one that follows the decisions from a full batch down to the floor to prove the retry loop terminates. The two single-code predicates it replaces are gone rather than left behind unused. Co-Authored-By: Claude Opus 5 --- src-tauri/src/event_log/fetch.rs | 121 +++++++++++++++++++++++++++++++ src-tauri/src/event_log/live.rs | 83 ++++++++++----------- src-tauri/src/event_log/mod.rs | 1 + 3 files changed, 159 insertions(+), 46 deletions(-) create mode 100644 src-tauri/src/event_log/fetch.rs diff --git a/src-tauri/src/event_log/fetch.rs b/src-tauri/src/event_log/fetch.rs new file mode 100644 index 000000000..d9a361431 --- /dev/null +++ b/src-tauri/src/event_log/fetch.rs @@ -0,0 +1,121 @@ +//! What to do when a batched fetch from the Event Log service fails. +//! +//! The decision lives here, apart from the call that produces the error, because everything around +//! it is Windows-only and therefore untestable anywhere else. Getting this wrong is expensive in a +//! specific way: the previous behaviour treated every failure as the end of the channel, so a +//! service that refused one request returned a partial channel that the caller reported as whole. +//! That is not a crash, it is a quiet wrong answer, and only a test that can run everywhere will +//! keep catching it. + +/// Windows `ERROR_NO_MORE_ITEMS`: the channel is exhausted. Not a failure. +const NO_MORE_ITEMS: u32 = 259; + +/// Windows `RPC_S_INVALID_BOUND`: the service refused the size of the request. +/// +/// Seen from `EvtNext` with a 256-handle batch, on one channel out of roughly twelve hundred on a +/// real machine. It says nothing about the channel's contents, so a smaller request is the right +/// answer rather than giving up on the channel. +const INVALID_BOUND: u32 = 1734; + +/// How a failed fetch should be handled. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FetchFailure { + /// The channel is fully read. Stop, and report nothing missing. + Exhausted, + /// Retry this fetch with a smaller batch. + RetryWith(usize), + /// Stop, and report that the channel was read only in part. + Truncated, +} + +/// Decides how to respond to a failed fetch of `batch` events. +/// +/// `floor` is the smallest batch worth attempting. Below it the request is as small as it is going +/// to get, and a further failure is about the channel rather than the request size. +pub fn classify_fetch_failure(win32_code: u32, batch: usize, floor: usize) -> FetchFailure { + if win32_code == NO_MORE_ITEMS { + return FetchFailure::Exhausted; + } + if win32_code == INVALID_BOUND && batch > floor { + // Halved rather than dropped straight to the floor, so a channel that can serve 128 is not + // read at 8 for the rest of the scan. + return FetchFailure::RetryWith((batch / 2).max(floor)); + } + FetchFailure::Truncated +} + +#[cfg(test)] +mod tests { + use super::*; + + const FLOOR: usize = 8; + + #[test] + fn an_exhausted_channel_is_not_a_failure() { + assert_eq!( + classify_fetch_failure(NO_MORE_ITEMS, 256, FLOOR), + FetchFailure::Exhausted + ); + } + + #[test] + fn a_refused_batch_size_is_retried_smaller() { + assert_eq!( + classify_fetch_failure(INVALID_BOUND, 256, FLOOR), + FetchFailure::RetryWith(128) + ); + } + + #[test] + fn halving_stops_at_the_floor_rather_than_reaching_zero() { + // A batch of zero would ask for no events and loop forever without reading anything. + assert_eq!( + classify_fetch_failure(INVALID_BOUND, 9, FLOOR), + FetchFailure::RetryWith(FLOOR) + ); + } + + #[test] + fn a_refusal_at_the_floor_is_reported_as_truncation() { + // The request is already as small as it gets, so the problem is not its size. Reporting + // Exhausted here would present a partly read channel as a complete one. + assert_eq!( + classify_fetch_failure(INVALID_BOUND, FLOOR, FLOOR), + FetchFailure::Truncated + ); + } + + #[test] + fn any_other_error_truncates_rather_than_looking_like_the_end_of_the_channel() { + // ERROR_ACCESS_DENIED partway through a channel is the case that matters: the events after + // it are missing, and calling that "exhausted" is the silent wrong answer. + for code in [5u32, 87, 1500, 0] { + assert_eq!( + classify_fetch_failure(code, 256, FLOOR), + FetchFailure::Truncated, + "win32 {code} must not be mistaken for the end of the channel" + ); + } + } + + #[test] + fn repeated_halving_walks_down_to_the_floor_and_then_stops() { + // The loop this models must terminate. Following the decisions from a full batch has to + // reach Truncated in a bounded number of steps rather than retrying forever. + let mut batch = 256; + let mut steps = 0; + loop { + match classify_fetch_failure(INVALID_BOUND, batch, FLOOR) { + FetchFailure::RetryWith(next) => { + assert!(next < batch, "a retry must shrink the request"); + batch = next; + } + FetchFailure::Truncated => break, + FetchFailure::Exhausted => panic!("a refusal is not an exhausted channel"), + } + steps += 1; + assert!(steps < 20, "halving should reach the floor quickly"); + } + assert_eq!(batch, FLOOR); + } +} diff --git a/src-tauri/src/event_log/live.rs b/src-tauri/src/event_log/live.rs index b52e1775b..e913cfa18 100644 --- a/src-tauri/src/event_log/live.rs +++ b/src-tauri/src/event_log/live.rs @@ -239,7 +239,7 @@ fn query_channel_inner( let mut raw_handles = [0isize; EVENT_FETCH_BATCH]; let mut returned = 0u32; - match unsafe { + let fetched = unsafe { EvtNext( query_handle.raw(), &mut raw_handles[..batch], @@ -247,37 +247,40 @@ fn query_channel_inner( 0, &mut returned, ) - } { - Ok(()) => {} - Err(e) if is_no_more_items(&e) => break, - // The service can refuse a batch this large on a particular channel. Halving and - // retrying reads it; the previous behaviour was to stop and report what had already - // been read as the channel's full contents. - Err(e) if is_invalid_bound(&e) && batch > MIN_FETCH_BATCH => { - batch = (batch / 2).max(MIN_FETCH_BATCH); - log::info!( - "event=evtx_batch_reduced channel=\"{channel}\" batch={batch} \ - reason=\"the service rejected the previous batch size\"" - ); - continue; - } - Err(e) => { - // Recorded as a gap, not just logged. The records already read are still returned, - // because they are real, but the channel must not be presented as complete. - let detail = format!( - "{channel}: stopped after {} events, the channel could not be read further ({}, 0x{:08x})", - records.len(), - e.message().trim(), - e.code().0 as u32 - ); - log::warn!( - "event=evtx_next_failed channel=\"{channel}\" batch={batch} \ - w32={} code=0x{:08x}", - win32_code(&e), - e.code().0 as u32 - ); - gaps.push(detail); - break; + }; + + if let Err(error) = fetched { + // How to respond is decided in `super::fetch`, where it is tested on every platform. + // Everything else in this loop is Windows-only, so a rule encoded here is a rule CI + // cannot check on any runner. + match super::fetch::classify_fetch_failure(win32_code(&error), batch, MIN_FETCH_BATCH) { + super::fetch::FetchFailure::Exhausted => break, + super::fetch::FetchFailure::RetryWith(smaller) => { + batch = smaller; + log::info!( + "event=evtx_batch_reduced channel=\"{channel}\" batch={batch} \ + reason=\"the service rejected the previous batch size\"" + ); + continue; + } + super::fetch::FetchFailure::Truncated => { + // Recorded as a gap, not only logged. The records already read are still + // returned because they are real, but the channel must not be presented as + // complete when the rest of it was never fetched. + log::warn!( + "event=evtx_next_failed channel=\"{channel}\" batch={batch} \ + w32={} code=0x{:08x}", + win32_code(&error), + error.code().0 as u32 + ); + gaps.push(format!( + "{channel}: stopped after {} events, the channel could not be read further ({}, 0x{:08x})", + records.len(), + error.message().trim(), + error.code().0 as u32 + )); + break; + } } } @@ -513,20 +516,8 @@ fn is_insufficient_buffer(error: &Error) -> bool { win32_code(error) == 122 } -#[cfg(target_os = "windows")] -fn is_no_more_items(error: &Error) -> bool { - win32_code(error) == 259 -} - -/// `RPC_S_INVALID_BOUND`: the service refused the size of the request, not its contents. -/// -/// Observed from `EvtNext` with a 256-handle batch on a real machine, on one channel out of roughly -/// twelve hundred. It says nothing about the channel's data, so the right response is a smaller -/// request rather than abandoning the channel. -#[cfg(target_os = "windows")] -fn is_invalid_bound(error: &Error) -> bool { - win32_code(error) == 1734 -} +// `ERROR_NO_MORE_ITEMS` and `RPC_S_INVALID_BOUND` are recognised in `super::fetch`, which owns the +// decision they feed and is tested on every platform rather than only on this one. #[cfg(target_os = "windows")] fn is_not_found(error: &Error) -> bool { diff --git a/src-tauri/src/event_log/mod.rs b/src-tauri/src/event_log/mod.rs index f3c9e3dc0..738e5d22e 100644 --- a/src-tauri/src/event_log/mod.rs +++ b/src-tauri/src/event_log/mod.rs @@ -4,6 +4,7 @@ pub mod export; pub mod maps; pub mod models; pub mod parser; +pub mod fetch; pub mod provider_db; pub mod rendered; pub mod timeline; From 41b8ec6a43f36b410d47b99ad2acec7cca99507b Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 12 Aug 2026 08:56:53 -0400 Subject: [PATCH 78/85] feat(event-log): deliver a channel's records as they are read One channel is most of a scan. Security measured 286,401 of 404,769 events and 191.8 seconds of a 267 second scan, so querying channels concurrently and delivering them one at a time - both already done - do nothing for it. An operator waited three minutes on a single blocking call with an empty list. query_channel_inner now hands each fetched batch to a callback as it is built. A caller that takes the records never holds more than one batch; a caller that ignores the argument gets the channel whole, as before, so the collecting wrappers and their tests are unchanged. ChannelScan carries `delivered` separately from `records`, because a streaming caller empties the vector and counting its length would report a fully read channel as holding nothing. The command emits each batch with a per-channel sequence number, and the store assembles the view from them. An event channel promises no delivery, so the store checks what it assembled against the count the reply states and against the sequence run, and reports either shortfall as a coverage gap. Without that check a dropped batch is indistinguishable from events that never happened, which is the failure this workspace exists to avoid. totalRecords moves into the validated reply shape for the same reason, with an absent count staying distinguishable from zero. Co-Authored-By: Claude Opus 5 --- src-tauri/src/event_log/commands.rs | 67 +++++++- src-tauri/src/event_log/live.rs | 76 +++++++-- src/workspaces/event-log/evtx-coverage.ts | 18 ++- .../event-log/evtx-store-coverage.test.ts | 150 +++++++++++++++++- src/workspaces/event-log/evtx-store.ts | 88 +++++++++- 5 files changed, 377 insertions(+), 22 deletions(-) diff --git a/src-tauri/src/event_log/commands.rs b/src-tauri/src/event_log/commands.rs index 89227bb7d..690a4ac02 100644 --- a/src-tauri/src/event_log/commands.rs +++ b/src-tauri/src/event_log/commands.rs @@ -16,6 +16,24 @@ struct EvtxQueryProgress { fetched: usize, } +/// A batch of records on its way to the frontend before the query has finished. +/// +/// One channel can be most of a scan: Security measured 286,401 of 404,769 events and 191.8 seconds +/// of 267, so a reply that waits for the channel to finish leaves an operator watching an empty +/// list for three minutes. Batches are emitted as they are read instead. +/// +/// `sequence` numbers the batches for one channel from zero. The receiver uses it to notice a batch +/// it never got: an event channel offers no delivery guarantee, and events that quietly failed to +/// arrive would look exactly like events that do not exist. +#[cfg(target_os = "windows")] +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct EvtxRecordBatch { + channel: String, + sequence: usize, + records: Vec, +} + #[tauri::command] pub async fn evtx_parse_files( paths: Vec, @@ -78,7 +96,9 @@ pub async fn evtx_query_channels( .map(|channel| { let app_ref = &app; let ch_name = channel.clone(); - let outcome = super::live::query_channel_filtered_with_progress( + let batch_channel = channel.clone(); + let mut sequence = 0usize; + let outcome = super::live::query_channel_streamed( channel, &query_filter, &maps, @@ -92,6 +112,32 @@ pub async fn evtx_query_channels( }, ); }, + |batch| { + if batch.is_empty() { + return; + } + // Taken from the batch rather than copied out of it. Leaving the records + // behind would send them and also accumulate them, which is the memory + // this exists to avoid. + let records = std::mem::take(batch); + let emitted = app_ref.emit( + "evtx-record-batch", + EvtxRecordBatch { + channel: batch_channel.clone(), + sequence, + records, + }, + ); + sequence += 1; + if let Err(error) = emitted { + // The records are gone: they were moved into the payload. Say so + // rather than letting the count quietly disagree with the view. + log::warn!( + "event=evtx_batch_emit_failed channel=\"{batch_channel}\" \ + sequence={sequence} error=\"{error}\"" + ); + } + }, ); (channel.clone(), outcome) }) @@ -101,15 +147,23 @@ pub async fn evtx_query_channels( let mut channel_infos = Vec::new(); let mut parse_errors = 0u32; let mut error_messages = Vec::new(); + // Emitted in batches rather than carried in this reply. Reported so the receiver can + // check what it actually assembled against what was sent, instead of assuming an event + // channel delivered everything. + let mut streamed = 0usize; for (channel, outcome) in per_channel { match outcome { Ok(scan) => { + // `delivered`, not `records.len()`: the records were emitted in batches and + // taken out of the vector on the way. Counting what is left would report a + // fully read channel as empty. channel_infos.push(super::models::EvtxChannelInfo { name: channel.clone(), - event_count: scan.records.len() as u64, + event_count: scan.delivered as u64, source_type: super::models::ChannelSourceType::Live, }); + streamed += scan.delivered; // A channel can be read partly. Those gaps travel with the records so a // truncated channel is not presented as a complete one, which is what // happened while the query returned records alone. @@ -138,12 +192,11 @@ pub async fn evtx_query_channels( } } + // `all_records` holds only what the batch hook left behind, which is nothing while the + // records are being streamed. `total_records` counts what was sent, so the receiver has + // something to check its own tally against. all_records.sort_by_key(|r| r.timestamp_epoch); - for (i, record) in all_records.iter_mut().enumerate() { - record.id = i as u64; - } - - let total_records = all_records.len() as u64; + let total_records = (streamed + all_records.len()) as u64; Ok(EvtxParseResult { records: all_records, diff --git a/src-tauri/src/event_log/live.rs b/src-tauri/src/event_log/live.rs index e913cfa18..e190843d2 100644 --- a/src-tauri/src/event_log/live.rs +++ b/src-tauri/src/event_log/live.rs @@ -38,7 +38,14 @@ const MIN_FETCH_BATCH: usize = 8; /// the events, and showed a channel that looked fully loaded. Events that were never fetched are /// indistinguishable on screen from events that do not exist. pub struct ChannelScan { + /// Records the caller did not take. Empty for a caller that streamed every batch away. pub records: Vec, + /// How many records this channel produced in total, whether or not the caller kept them. + /// + /// Separate from `records.len()` because a streaming caller empties that vector as it goes. + /// Reporting the length instead would tell the frontend a fully read channel held no events, + /// which is the same wrong answer as a channel that failed. + pub delivered: usize, /// Operator-facing explanations of what is missing. Empty means the channel was read whole. pub gaps: Vec, } @@ -165,7 +172,7 @@ pub fn query_channel_filtered( maps: &MapRegistry, max_events: Option, ) -> Result { - query_channel_inner(channel, filter, maps, max_events, |_, _| {}) + query_channel_inner(channel, filter, maps, max_events, |_, _| {}, |_| {}) } /// Query with a progress callback: `on_progress(fetched_so_far, total_estimate)`. @@ -182,6 +189,7 @@ pub fn query_channel_with_progress( maps, max_events, on_progress, + |_| {}, ) } @@ -194,9 +202,36 @@ pub fn query_channel_filtered_with_progress( max_events: Option, on_progress: impl Fn(usize, Option), ) -> Result { - query_channel_inner(channel, filter, maps, max_events, on_progress) + query_channel_inner(channel, filter, maps, max_events, on_progress, |_| {}) } +/// Queries a channel, delivering each batch of records as it is read. +/// +/// `on_batch` is handed every batch and is expected to take the records from it. Whatever it leaves +/// is returned in the [`ChannelScan`], so a caller that forgets to drain still gets correct results +/// rather than losing them; it simply holds the channel in memory as before. +#[cfg(target_os = "windows")] +pub fn query_channel_streamed( + channel: &str, + filter: &EventQueryFilter, + maps: &MapRegistry, + max_events: Option, + on_progress: impl Fn(usize, Option), + on_batch: impl FnMut(&mut Vec), +) -> Result { + query_channel_inner(channel, filter, maps, max_events, on_progress, on_batch) +} + +/// Reads a channel, handing each fetched batch to `on_batch` as it is built. +/// +/// `on_batch` receives the batch by mutable reference and may take the records out of it. Whatever +/// it leaves behind is accumulated into the returned [`ChannelScan`]. That is the whole difference +/// between streaming and collecting: a caller that drains never holds more than one batch, and a +/// caller that ignores the argument gets the channel in one piece exactly as before. +/// +/// The distinction matters because one channel dominates a scan. On a measured seven-day scan, +/// Security was 286,401 of 404,769 events and 191.8 seconds of 267, so a caller waiting for this +/// function to return waits three minutes with nothing to show. #[cfg(target_os = "windows")] fn query_channel_inner( channel: &str, @@ -204,6 +239,7 @@ fn query_channel_inner( maps: &MapRegistry, max_events: Option, on_progress: impl Fn(usize, Option), + mut on_batch: impl FnMut(&mut Vec), ) -> Result { let limit = max_events.map(|n| n as usize).unwrap_or(usize::MAX); let channel_hstring = HSTRING::from(channel); @@ -234,8 +270,12 @@ fn query_channel_inner( let mut gaps = Vec::new(); let mut batch = EVENT_FETCH_BATCH; + // Counted separately from `records`, which a streaming caller empties as it goes. Using the + // length of a vector the caller is allowed to drain would restart the limit at zero after every + // batch and read the channel forever. + let mut produced = 0usize; - while records.len() < limit { + while produced < limit { let mut raw_handles = [0isize; EVENT_FETCH_BATCH]; let mut returned = 0u32; @@ -275,7 +315,7 @@ fn query_channel_inner( ); gaps.push(format!( "{channel}: stopped after {} events, the channel could not be read further ({}, 0x{:08x})", - records.len(), + produced, error.message().trim(), error.code().0 as u32 )); @@ -288,8 +328,14 @@ fn query_channel_inner( break; } + // Built per fetch rather than appended straight to `records`, so the caller can take each + // batch as it is produced. A caller that takes them holds nothing here, which is what keeps + // a channel the size of Security from occupying its whole result set before anything is + // shown. + let mut batch_records: Vec = Vec::new(); + for raw_handle in raw_handles.into_iter().take(returned as usize) { - if records.len() >= limit { + if produced + batch_records.len() >= limit { // Close remaining handles we won't use unsafe { let _ = EvtClose(EVT_HANDLE(raw_handle)); @@ -329,7 +375,7 @@ fn query_channel_inner( .flatten() }); - records.push(super::rendered::record_from_parts( + batch_records.push(super::rendered::record_from_parts( &parsed, system, &xml, @@ -337,11 +383,15 @@ fn query_channel_inner( maps, rendered_message.as_deref(), )); - // Report progress every 100 records - if records.len() % 100 == 0 { - on_progress(records.len(), None); - } } + + produced += batch_records.len(); + on_progress(produced, None); + + // The caller sees the batch before anything else happens to it. Draining it here is what + // makes delivery incremental; leaving it collects the channel as before. + on_batch(&mut batch_records); + records.append(&mut batch_records); } if unparsable > 0 { @@ -357,7 +407,11 @@ fn query_channel_inner( records.len(), gaps.len() ); - Ok(ChannelScan { records, gaps }) + Ok(ChannelScan { + records, + delivered: produced, + gaps, + }) } // ── Non-Windows stubs ─────────────────────────────────────────────────────── diff --git a/src/workspaces/event-log/evtx-coverage.ts b/src/workspaces/event-log/evtx-coverage.ts index 0d68bf93a..e049c390c 100644 --- a/src/workspaces/event-log/evtx-coverage.ts +++ b/src/workspaces/event-log/evtx-coverage.ts @@ -43,8 +43,14 @@ export function assertParseResultShape(value: unknown): { records: unknown[]; channels: unknown[]; errorMessages: string[]; + totalRecords: number | null; } { - const reply = value as { records?: unknown; channels?: unknown; errorMessages?: unknown }; + const reply = value as { + records?: unknown; + channels?: unknown; + errorMessages?: unknown; + totalRecords?: unknown; + }; if (!Array.isArray(reply?.records) || !Array.isArray(reply?.channels)) { throw new Error("the event log reader returned a reply this build cannot read"); } @@ -55,5 +61,15 @@ export function assertParseResultShape(value: unknown): { errorMessages: Array.isArray(reply.errorMessages) ? reply.errorMessages.filter((entry): entry is string => typeof entry === "string") : [], + // How many records the reader says it sent, counting any streamed separately from this reply. + // `null` when the reader did not say, which must stay distinguishable from zero: treating an + // absent count as zero would turn "I cannot check completeness" into "nothing was missing". + // A non-finite or negative count is no answer either, so it is rejected the same way. + totalRecords: + typeof reply.totalRecords === "number" && + Number.isFinite(reply.totalRecords) && + reply.totalRecords >= 0 + ? reply.totalRecords + : null, }; } diff --git a/src/workspaces/event-log/evtx-store-coverage.test.ts b/src/workspaces/event-log/evtx-store-coverage.test.ts index 37ff8d5fd..e7d967cd5 100644 --- a/src/workspaces/event-log/evtx-store-coverage.test.ts +++ b/src/workspaces/event-log/evtx-store-coverage.test.ts @@ -10,11 +10,25 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const invoke = vi.hoisted(() => vi.fn()); + +// The store subscribes to backend events at module scope. Capturing the handlers lets a test +// deliver a batch exactly as the backend would, including delivering none. +const listeners = vi.hoisted(() => new Map void>()); vi.mock("@tauri-apps/api/core", () => ({ invoke })); -vi.mock("@tauri-apps/api/event", () => ({ listen: vi.fn().mockResolvedValue(() => {}) })); +vi.mock("@tauri-apps/api/event", () => ({ + listen: vi.fn((name: string, handler: (event: { payload: unknown }) => void) => { + listeners.set(name, handler); + return Promise.resolve(() => {}); + }), +})); const { useEvtxStore } = await import("./evtx-store"); +/** Delivers a batch the way the backend emits one. */ +function emitBatch(channel: string, sequence: number, records: unknown[]) { + listeners.get("evtx-record-batch")?.({ payload: { channel, sequence, records } }); +} + function result(channel: string, gaps: string[]) { return { records: [], @@ -249,3 +263,137 @@ describe("the time window reaches the service", () => { }); }); }); + +describe("records that arrive in batches while the query runs", () => { + beforeEach(() => { + invoke.mockReset(); + useEvtxStore.setState({ + records: [], + channels: [], + coverageGaps: [], + loadedChannels: new Set(), + selectedChannels: new Set(), + }); + }); + + function record(channel: string, epoch: number) { + return { + id: 0, + eventRecordId: epoch, + timestamp: "2026-08-12T12:00:00.000Z", + timestampEpoch: epoch, + provider: "P", + channel, + eventId: 1, + level: "information", + computer: "C", + message: "m", + eventData: [], + rawXml: "", + sourceLabel: "Live", + mapped: null, + }; + } + + /** A reply that streamed everything: it carries the count but none of the records. */ + function streamedReply(channel: string, totalRecords: number) { + return { + records: [], + channels: [{ name: channel, eventCount: totalRecords, sourceType: "live" }], + totalRecords, + parseErrors: 0, + errorMessages: [], + }; + } + + it("assembles the view from batches the reply did not carry", async () => { + invoke.mockImplementationOnce(async () => { + emitBatch("System", 0, [record("System", 1), record("System", 2)]); + emitBatch("System", 1, [record("System", 3)]); + return streamedReply("System", 3); + }); + + await useEvtxStore.getState().queryChannels(["System"]); + + const state = useEvtxStore.getState(); + expect(state.records).toHaveLength(3); + expect(state.coverageGaps).toEqual([]); + }); + + it("reports a batch that never arrived instead of showing a short list as complete", async () => { + // Sequence 1 is skipped. Its events are simply absent, and an absent event is indistinguishable + // from an event that never happened unless the gap is stated. + invoke.mockImplementationOnce(async () => { + emitBatch("System", 0, [record("System", 1)]); + emitBatch("System", 2, [record("System", 3)]); + return streamedReply("System", 3); + }); + + await useEvtxStore.getState().queryChannels(["System"]); + + const gaps = useEvtxStore.getState().coverageGaps; + expect(gaps.some((g) => g.includes("System") && g.includes("batches"))).toBe(true); + }); + + it("reports a shortfall against the count the reader sent", async () => { + // Every batch arrived in order, but fewer events than the reader says it sent. + invoke.mockImplementationOnce(async () => { + emitBatch("System", 0, [record("System", 1)]); + return streamedReply("System", 9); + }); + + await useEvtxStore.getState().queryChannels(["System"]); + + const gaps = useEvtxStore.getState().coverageGaps; + expect(gaps.some((g) => g.includes("8 of 9"))).toBe(true); + }); + + it("does not invent a shortfall when the reader gave no count", async () => { + // An absent count means completeness cannot be checked. Treating it as zero would report every + // arriving record as unexpected; treating it as a shortfall would cry wolf on every load. + invoke.mockImplementationOnce(async () => { + emitBatch("System", 0, [record("System", 1)]); + return { ...streamedReply("System", 0), totalRecords: "unknown" }; + }); + + await useEvtxStore.getState().queryChannels(["System"]); + + expect(useEvtxStore.getState().records).toHaveLength(1); + expect(useEvtxStore.getState().coverageGaps).toEqual([]); + }); + + it("still works when the reader returned the records in the reply instead", async () => { + // Collecting callers exist, and a backend that did not stream must not look like a total loss. + invoke.mockResolvedValueOnce({ + records: [record("System", 1), record("System", 2)], + channels: [{ name: "System", eventCount: 2, sourceType: "live" }], + totalRecords: 2, + parseErrors: 0, + errorMessages: [], + }); + + await useEvtxStore.getState().queryChannels(["System"]); + + expect(useEvtxStore.getState().records).toHaveLength(2); + expect(useEvtxStore.getState().coverageGaps).toEqual([]); + }); + + it("does not count a previous attempt's batches towards a retry", async () => { + invoke.mockImplementationOnce(async () => { + emitBatch("System", 0, [record("System", 1)]); + throw new Error("interrupted"); + }); + await useEvtxStore.getState().queryChannels(["System"]); + + useEvtxStore.setState({ records: [], coverageGaps: [], loadedChannels: new Set() }); + invoke.mockImplementationOnce(async () => { + emitBatch("System", 0, [record("System", 5)]); + return streamedReply("System", 1); + }); + await useEvtxStore.getState().queryChannels(["System"]); + + const state = useEvtxStore.getState(); + expect(state.records).toHaveLength(1); + expect(state.records[0].eventRecordId).toBe(5); + }); +}); diff --git a/src/workspaces/event-log/evtx-store.ts b/src/workspaces/event-log/evtx-store.ts index 29d00bae2..d67026833 100644 --- a/src/workspaces/event-log/evtx-store.ts +++ b/src/workspaces/event-log/evtx-store.ts @@ -263,6 +263,10 @@ export const useEvtxStore = create()((set, get) => ({ // away the results of every channel queried alongside it and left the view empty. let loadError: string | null = null; + // Anything left over from an earlier attempt at these channels is dropped, so a retry cannot + // count a previous run's batches towards this one. + resetStreamedRecords(channels); + const results = await Promise.all( channels.map(async (ch) => { try { @@ -295,10 +299,32 @@ export const useEvtxStore = create()((set, get) => ({ } const checked = assertParseResultShape(result); + + // The records travel as batches while the query runs; the reply carries only whatever the + // backend did not stream. Both are taken, so this works whether or not streaming happened. + const streamed = drainStreamedRecords(channel); + const arrived = [...streamed.records, ...result.records]; + + // The reply says how many records were sent. Silence is not agreement: if fewer arrived, or a + // batch number is missing from the run, those events are absent from the view and must be + // said so rather than left to look like events that never happened. + const gapsFound: string[] = []; + if (streamed.missingSequences.length > 0) { + gapsFound.push( + `${channel}: ${streamed.missingSequences.length} batches of events were not received` + ); + } + const expected = checked.totalRecords; + if (typeof expected === "number" && arrived.length < expected) { + gapsFound.push( + `${channel}: ${expected - arrived.length} of ${expected} events did not reach the view` + ); + } + const state = get(); const existingChannelNames = new Set(state.records.map((r) => r.channel)); // Only add records from channels we don't already have - const newRecords = result.records.filter((r) => !existingChannelNames.has(r.channel)); + const newRecords = arrived.filter((r) => !existingChannelNames.has(r.channel)); const merged = [...state.records, ...newRecords]; merged.sort((a, b) => a.timestampEpoch - b.timestampEpoch); // Reassign IDs @@ -320,7 +346,10 @@ export const useEvtxStore = create()((set, get) => ({ loadedChannels: newLoaded, // Accumulated, not dropped. This path loads channels incrementally, so discarding what the // backend reported here would show a complete view of a partly unreadable set. - coverageGaps: mergeCoverageGaps(state.coverageGaps, checked.errorMessages), + coverageGaps: mergeCoverageGaps(state.coverageGaps, [ + ...checked.errorMessages, + ...gapsFound, + ]), }); } @@ -497,3 +526,58 @@ listen<{ channel: string; fetched: number }>("evtx-query-progress", (event) => { }); }); +/** + * Records arriving in batches while a query is still running. + * + * A channel can be most of a scan on its own: Security measured 286,401 of 404,769 events and 191.8 + * seconds of 267, so waiting for the reply meant three minutes of empty list. Batches are collected + * here as they arrive and drained by the query that asked for them. + * + * Keyed by channel, and the sequence numbers are kept rather than discarded. An event channel makes + * no delivery promise, so the query checks both the count and the sequence run before treating a + * channel as complete. A batch that never arrived would otherwise be indistinguishable from events + * that do not exist, which is the failure this workspace exists to avoid. + */ +const pendingBatches = new Map }>(); + +listen<{ channel: string; sequence: number; records: EvtxRecord[] }>( + "evtx-record-batch", + (event) => { + const { channel, sequence, records } = event.payload; + let pending = pendingBatches.get(channel); + if (!pending) { + pending = { records: [], sequences: new Set() }; + pendingBatches.set(channel, pending); + } + // A repeated sequence is counted once. Appending it twice would inflate the tally and hide a + // batch that really is missing. + if (pending.sequences.has(sequence)) return; + pending.sequences.add(sequence); + pending.records.push(...records); + } +); + +/** Takes everything received for `channel` so far, and reports whether it is contiguous. */ +export function drainStreamedRecords(channel: string): { + records: EvtxRecord[]; + missingSequences: number[]; +} { + const pending = pendingBatches.get(channel); + pendingBatches.delete(channel); + if (!pending) return { records: [], missingSequences: [] }; + + // Batches are numbered from zero, so any number below the highest that never arrived is a batch + // whose records are simply absent. + const highest = Math.max(...pending.sequences); + const missingSequences: number[] = []; + for (let i = 0; i < highest; i++) { + if (!pending.sequences.has(i)) missingSequences.push(i); + } + return { records: pending.records, missingSequences }; +} + +/** Discards anything buffered for channels a query is about to start, so a retry cannot double up. */ +export function resetStreamedRecords(channels: string[]) { + for (const channel of channels) pendingBatches.delete(channel); +} + From f008ac9a3c1d0f942acf37cdbe6086c90c4f6b7d Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 12 Aug 2026 18:56:07 -0400 Subject: [PATCH 79/85] fix(event-log): count render failures and slice previews by character A single handle that failed EvtRender propagated Err for the whole channel, discarding every record already read; the caller then reported the channel as holding no events. It is now counted as an unrenderable gap and the scan continues, matching how an unparsable document is already treated. The unparsable-warning preview sliced the XML by byte offset, which panics when byte 300 lands inside a multi-byte character. It is now sliced by character. --- src-tauri/src/event_log/live.rs | 34 ++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/src-tauri/src/event_log/live.rs b/src-tauri/src/event_log/live.rs index e190843d2..b5123a5b8 100644 --- a/src-tauri/src/event_log/live.rs +++ b/src-tauri/src/event_log/live.rs @@ -267,6 +267,7 @@ fn query_channel_inner( let mut records = Vec::new(); let mut publisher_metadata = HashMap::>::new(); let mut unparsable = 0usize; + let mut unrenderable = 0usize; let mut gaps = Vec::new(); let mut batch = EVENT_FETCH_BATCH; @@ -344,8 +345,22 @@ fn query_channel_inner( } let event_handle = OwnedEvtHandle::new(EVT_HANDLE(raw_handle)); - let xml = - render_event_xml(event_handle.raw()).map_err(|e| format_error("EvtRender", &e))?; + // A handle that fails to render is counted, not fatal. Propagating it here returned + // Err for the whole channel and threw away every record already read, which the caller + // then reported as a channel with no events. + let xml = match render_event_xml(event_handle.raw()) { + Ok(xml) => xml, + Err(error) => { + unrenderable += 1; + if unrenderable == 1 { + log::warn!( + "event=evtx_render_failed channel=\"{channel}\" error=\"{}\"", + format_error("EvtRender", &error) + ); + } + continue; + } + }; // Parsed once here and handed to the record builder. The provider has to be known // before the record exists, because it names the publisher whose message template the @@ -356,9 +371,12 @@ fn query_channel_inner( Err(error) => { unparsable += 1; if unparsable == 1 { + // Sliced by character rather than by byte. A byte cut that lands inside a + // multi-byte character panics, and this XML carries account names and paths + // that are routinely not ASCII. + let prefix: String = xml.chars().take(300).collect(); log::warn!( - "event=evtx_parse_failed channel=\"{channel}\" error=\"{error}\" xml_prefix=\"{}\"", - &xml[..xml.len().min(300)] + "event=evtx_parse_failed channel=\"{channel}\" error=\"{error}\" xml_prefix=\"{prefix}\"" ); } continue; @@ -402,8 +420,14 @@ fn query_channel_inner( "{channel}: {unparsable} events could not be read and are missing from this view" )); } + if unrenderable > 0 { + log::warn!("event=evtx_live_query_gap channel=\"{channel}\" unrenderable={unrenderable}"); + gaps.push(format!( + "{channel}: {unrenderable} events could not be rendered and are missing from this view" + )); + } log::info!( - "event=evtx_live_query_done channel=\"{channel}\" records={} unparsable={unparsable} gaps={}", + "event=evtx_live_query_done channel=\"{channel}\" records={} unparsable={unparsable} unrenderable={unrenderable} gaps={}", records.len(), gaps.len() ); From 6a47c0fcd61684352e44cb0157d1ac3fa80a17c5 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 12 Aug 2026 18:56:11 -0400 Subject: [PATCH 80/85] fix(event-log): build the file path's summary the way the live path does The file path fell back to its own build_message, which never truncated long field values, while the live path used build_event_data_summary, which does. The same event therefore rendered a different message depending on how it was opened. The file path now shares the live path's summary and build_message is gone. --- src-tauri/src/event_log/parser.rs | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/src-tauri/src/event_log/parser.rs b/src-tauri/src/event_log/parser.rs index 1e102d7bc..a1bc48412 100644 --- a/src-tauri/src/event_log/parser.rs +++ b/src-tauri/src/event_log/parser.rs @@ -231,7 +231,7 @@ fn parse_single_file( // every other cross-platform reader shows and why they are hard to read. let message = describe_event(&providers, &provider, event_id, &insertions) .or(payload) - .unwrap_or_else(|| build_message(&fields)); + .unwrap_or_else(|| super::rendered::build_event_data_summary(&fields)); let mapped = super::maps::apply_registered(&maps, &channel, &provider, event_id, &parsed); records.push(EvtxRecord { @@ -307,16 +307,6 @@ fn describe_event( } } -/// Build a human-readable message from the first few EventData fields. -fn build_message(event_data: &[EvtxField]) -> String { - event_data - .iter() - .take(5) - .map(|f| format!("{}: {}", f.name, f.value)) - .collect::>() - .join("; ") -} - #[cfg(test)] mod tests { use super::*; @@ -576,22 +566,6 @@ mod tests { 1_786_276_800_000 ); } - - #[test] - fn test_build_message() { - let fields = vec![ - EvtxField { - name: "Key1".into(), - value: "Val1".into(), - }, - EvtxField { - name: "Key2".into(), - value: "Val2".into(), - }, - ]; - let msg = build_message(&fields); - assert_eq!(msg, "Key1: Val1; Key2: Val2"); - } } #[cfg(test)] From 9b7c5a437bbe64d1ebaa805fb2d2293d43dc9d1a Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 12 Aug 2026 18:56:15 -0400 Subject: [PATCH 81/85] fix(event-log): guard channel processing and bound the sequence scan The per-channel processing loop after the invoke call was unguarded, so a malformed reply (assertParseResultShape throws by design) or a drain failure rejected queryChannels before isLoading was cleared, leaving a stuck spinner with no message. Each iteration is now wrapped so the failure becomes a gap and a load error. drainStreamedRecords spread the whole sequence set into Math.max(...), which throws RangeError once a channel produces more batches than the engine accepts as arguments. The highest sequence is now found by reduction. --- .../event-log/evtx-store-coverage.test.ts | 31 ++++ src/workspaces/event-log/evtx-store.ts | 143 ++++++++++-------- 2 files changed, 111 insertions(+), 63 deletions(-) diff --git a/src/workspaces/event-log/evtx-store-coverage.test.ts b/src/workspaces/event-log/evtx-store-coverage.test.ts index e7d967cd5..7344f21e7 100644 --- a/src/workspaces/event-log/evtx-store-coverage.test.ts +++ b/src/workspaces/event-log/evtx-store-coverage.test.ts @@ -396,4 +396,35 @@ describe("records that arrive in batches while the query runs", () => { expect(state.records).toHaveLength(1); expect(state.records[0].eventRecordId).toBe(5); }); + + it("finds the highest batch number without spreading every one", async () => { + // The highest sequence is found by reduction. Spreading the set into Math.max(...) arguments + // throws RangeError once a channel produces more batches than the engine accepts as arguments, + // which a reduced fetch batch makes reachable for a channel the size of Security. + const batches = 200_000; + invoke.mockImplementationOnce(async () => { + for (let sequence = 0; sequence < batches; sequence++) { + emitBatch("Huge", sequence, []); + } + return streamedReply("Huge", 0); + }); + + await useEvtxStore.getState().queryChannels(["Huge"]); + + expect(useEvtxStore.getState().coverageGaps).toEqual([]); + }); + + it("does not leave the workspace stuck loading when a reply is unreadable", async () => { + // assertParseResultShape throws by design on a reply this build cannot read. The processing + // loop was unguarded, so the throw rejected queryChannels before isLoading was cleared and the + // operator saw an endless spinner with no error. + invoke.mockResolvedValueOnce({ records: null, channels: [] }); + + await useEvtxStore.getState().queryChannels(["Application"]); + + const state = useEvtxStore.getState(); + expect(state.isLoading).toBe(false); + expect(state.loadError).toContain("Application"); + expect(state.coverageGaps.some((g) => g.includes("Application"))).toBe(true); + }); }); diff --git a/src/workspaces/event-log/evtx-store.ts b/src/workspaces/event-log/evtx-store.ts index d67026833..b4b042290 100644 --- a/src/workspaces/event-log/evtx-store.ts +++ b/src/workspaces/event-log/evtx-store.ts @@ -286,71 +286,83 @@ export const useEvtxStore = create()((set, get) => ({ ); for (const { channel, result, error } of results) { - if (!result) { - // A channel that could not be read is recorded as a gap, not merely as an error banner - // that the next successful load replaces. The events it would have contributed are absent - // from the view for as long as the view is on screen. - set((s) => ({ - coverageGaps: mergeCoverageGaps(s.coverageGaps, [ - `${channel}: not read (${error ?? "unknown error"})`, - ]), - })); - continue; - } + try { + if (!result) { + // A channel that could not be read is recorded as a gap, not merely as an error banner + // that the next successful load replaces. The events it would have contributed are absent + // from the view for as long as the view is on screen. + set((s) => ({ + coverageGaps: mergeCoverageGaps(s.coverageGaps, [ + `${channel}: not read (${error ?? "unknown error"})`, + ]), + })); + continue; + } - const checked = assertParseResultShape(result); + const checked = assertParseResultShape(result); - // The records travel as batches while the query runs; the reply carries only whatever the - // backend did not stream. Both are taken, so this works whether or not streaming happened. - const streamed = drainStreamedRecords(channel); - const arrived = [...streamed.records, ...result.records]; - - // The reply says how many records were sent. Silence is not agreement: if fewer arrived, or a - // batch number is missing from the run, those events are absent from the view and must be - // said so rather than left to look like events that never happened. - const gapsFound: string[] = []; - if (streamed.missingSequences.length > 0) { - gapsFound.push( - `${channel}: ${streamed.missingSequences.length} batches of events were not received` - ); - } - const expected = checked.totalRecords; - if (typeof expected === "number" && arrived.length < expected) { - gapsFound.push( - `${channel}: ${expected - arrived.length} of ${expected} events did not reach the view` - ); - } + // The records travel as batches while the query runs; the reply carries only whatever the + // backend did not stream. Both are taken, so this works whether or not streaming happened. + const streamed = drainStreamedRecords(channel); + const arrived = [...streamed.records, ...result.records]; + + // The reply says how many records were sent. Silence is not agreement: if fewer arrived, or a + // batch number is missing from the run, those events are absent from the view and must be + // said so rather than left to look like events that never happened. + const gapsFound: string[] = []; + if (streamed.missingSequences.length > 0) { + gapsFound.push( + `${channel}: ${streamed.missingSequences.length} batches of events were not received` + ); + } + const expected = checked.totalRecords; + if (typeof expected === "number" && arrived.length < expected) { + gapsFound.push( + `${channel}: ${expected - arrived.length} of ${expected} events did not reach the view` + ); + } - const state = get(); - const existingChannelNames = new Set(state.records.map((r) => r.channel)); - // Only add records from channels we don't already have - const newRecords = arrived.filter((r) => !existingChannelNames.has(r.channel)); - const merged = [...state.records, ...newRecords]; - merged.sort((a, b) => a.timestampEpoch - b.timestampEpoch); - // Reassign IDs - for (let i = 0; i < merged.length; i++) merged[i].id = i; - - // Update channel event counts - const countMap = new Map(result.channels.map((c) => [c.name, c.eventCount])); - const updatedChannels = state.channels.map((c) => ({ - ...c, - eventCount: countMap.get(c.name) ?? c.eventCount, - })); - - const newLoaded = new Set(state.loadedChannels); - newLoaded.add(channel); + const state = get(); + const existingChannelNames = new Set(state.records.map((r) => r.channel)); + // Only add records from channels we don't already have + const newRecords = arrived.filter((r) => !existingChannelNames.has(r.channel)); + const merged = [...state.records, ...newRecords]; + merged.sort((a, b) => a.timestampEpoch - b.timestampEpoch); + // Reassign IDs + for (let i = 0; i < merged.length; i++) merged[i].id = i; - set({ - records: merged, - channels: updatedChannels, - loadedChannels: newLoaded, - // Accumulated, not dropped. This path loads channels incrementally, so discarding what the - // backend reported here would show a complete view of a partly unreadable set. - coverageGaps: mergeCoverageGaps(state.coverageGaps, [ - ...checked.errorMessages, - ...gapsFound, - ]), - }); + // Update channel event counts + const countMap = new Map(result.channels.map((c) => [c.name, c.eventCount])); + const updatedChannels = state.channels.map((c) => ({ + ...c, + eventCount: countMap.get(c.name) ?? c.eventCount, + })); + + const newLoaded = new Set(state.loadedChannels); + newLoaded.add(channel); + + set({ + records: merged, + channels: updatedChannels, + loadedChannels: newLoaded, + // Accumulated, not dropped. This path loads channels incrementally, so discarding what the + // backend reported here would show a complete view of a partly unreadable set. + coverageGaps: mergeCoverageGaps(state.coverageGaps, [ + ...checked.errorMessages, + ...gapsFound, + ]), + }); + } catch (processingError) { + // assertParseResultShape throws by design on a reply this build cannot read, and a malformed + // reply is not a reason to leave the workspace stuck on a spinner with no message. + const message = + processingError instanceof Error ? processingError.message : String(processingError); + console.warn(`[evtx] Failed to process ${channel}: ${message}`); + if (!loadError) loadError = `${channel}: ${message}`; + set((s) => ({ + coverageGaps: mergeCoverageGaps(s.coverageGaps, [`${channel}: not read (${message})`]), + })); + } } set({ isLoading: false, loadError }); @@ -567,8 +579,13 @@ export function drainStreamedRecords(channel: string): { if (!pending) return { records: [], missingSequences: [] }; // Batches are numbered from zero, so any number below the highest that never arrived is a batch - // whose records are simply absent. - const highest = Math.max(...pending.sequences); + // whose records are simply absent. Reduced rather than spread: one argument per batch throws + // RangeError once a channel produces more batches than the engine accepts as arguments, which a + // reduced fetch batch makes reachable. + let highest = 0; + for (const sequence of pending.sequences) { + if (sequence > highest) highest = sequence; + } const missingSequences: number[] = []; for (let i = 0; i < highest; i++) { if (!pending.sequences.has(i)) missingSequences.push(i); From 2de996e6190616a925c35f49221e8c907072aaf0 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 12 Aug 2026 18:56:19 -0400 Subject: [PATCH 82/85] fix(event-log): reject a --days value that overflows the time window The harness multiplied days by milliseconds without a bound, so a large --days panicked in debug and scanned the wrong window in release. It now uses checked arithmetic and exits with an input error on overflow. --- src-tauri/examples/evtx_scan.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src-tauri/examples/evtx_scan.rs b/src-tauri/examples/evtx_scan.rs index cbdbe3c24..500713f84 100644 --- a/src-tauri/examples/evtx_scan.rs +++ b/src-tauri/examples/evtx_scan.rs @@ -53,10 +53,13 @@ fn run(days: u64, only_channel: Option, max_events: Option) { .collect(); let enumerate_ms = enumerated.elapsed().as_millis(); + let Some(milliseconds) = days.checked_mul(24 * 60 * 60 * 1000) else { + eprintln!("--days is too large"); + std::process::exit(2); + }; + let filter = EventQueryFilter { - time: Some(TimeWindow::Last { - milliseconds: days * 24 * 60 * 60 * 1000, - }), + time: Some(TimeWindow::Last { milliseconds }), ..Default::default() }; From ee5af284a7bb404a096b9c872635e20eb3b693d9 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 12 Aug 2026 19:16:59 -0400 Subject: [PATCH 83/85] fix(event-log): report a channel whose refresh failed as a gap refreshLoadedChannels cleared coverageGaps with the records it replaced, but its per-channel catch only logged. A channel whose refresh request failed then contributed zero records to the replaced view while it reported full coverage. The failure is now recorded as a gap and a load error, matching queryChannels. --- .../event-log/evtx-store-coverage.test.ts | 17 +++++++++++++++++ src/workspaces/event-log/evtx-store.ts | 9 ++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/workspaces/event-log/evtx-store-coverage.test.ts b/src/workspaces/event-log/evtx-store-coverage.test.ts index 7344f21e7..86af2475b 100644 --- a/src/workspaces/event-log/evtx-store-coverage.test.ts +++ b/src/workspaces/event-log/evtx-store-coverage.test.ts @@ -104,6 +104,23 @@ describe("coverage gaps through the store", () => { expect(useEvtxStore.getState().coverageGaps).toEqual(["Application: fresh gap"]); }); + + it("records a channel whose refresh failed instead of showing the cleared view as complete", async () => { + // The refresh clears coverageGaps with the records it replaced. A channel whose refresh then + // fails contributes zero records to the replaced view, so the failure must be recorded or the + // view reports full coverage while missing a whole channel. + invoke.mockResolvedValueOnce(result("Application", [])); + await useEvtxStore.getState().queryChannels(["Application"]); + + invoke.mockRejectedValueOnce(new Error("access denied")); + await useEvtxStore.getState().refreshLoadedChannels(); + + const state = useEvtxStore.getState(); + expect( + state.coverageGaps.some((g) => g.includes("Application") && g.includes("access denied")) + ).toBe(true); + expect(state.loadError).toContain("Application"); + }); }); describe("a multi-channel query is delivered one channel at a time", () => { diff --git a/src/workspaces/event-log/evtx-store.ts b/src/workspaces/event-log/evtx-store.ts index b4b042290..ef5b3b2c3 100644 --- a/src/workspaces/event-log/evtx-store.ts +++ b/src/workspaces/event-log/evtx-store.ts @@ -428,7 +428,14 @@ export const useEvtxStore = create()((set, get) => ({ coverageGaps: mergeCoverageGaps(s.coverageGaps, checked.errorMessages), }); } catch (e) { - console.warn(`[evtx] Refresh failed for ${ch}:`, e); + const message = e instanceof Error ? e.message : String(e); + console.warn(`[evtx] Refresh failed for ${ch}: ${message}`); + // Recorded, not only logged. The refresh cleared the previous gaps, so a silent failure + // here presents a view that is missing a whole channel as complete. + set((s) => ({ + coverageGaps: mergeCoverageGaps(s.coverageGaps, [`${ch}: not read (${message})`]), + loadError: s.loadError ?? `${ch}: ${message}`, + })); } }); From 1a631b6d1c13c873543cdf6b0e0c32fa59de2c99 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 12 Aug 2026 19:17:03 -0400 Subject: [PATCH 84/85] fix(event-log): count channels with gaps separately from gap entries The harness summed gap entries and printed the total as channels_with_gaps. A channel can report several gaps at once, so the number overstated how many channels came back incomplete. Channels and entries are now counted apart. --- src-tauri/examples/evtx_scan.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src-tauri/examples/evtx_scan.rs b/src-tauri/examples/evtx_scan.rs index 500713f84..cc1b7e6b4 100644 --- a/src-tauri/examples/evtx_scan.rs +++ b/src-tauri/examples/evtx_scan.rs @@ -78,8 +78,11 @@ fn run(days: u64, only_channel: Option, max_events: Option) { let mut widest_channel = (String::new(), 0usize); // A channel read only partly is neither a success nor a failure, and counting it as // either hides it. The scan's own truncations are the thing most likely to make a - // faster number look like an improvement. + // faster number look like an improvement. Gap entries and affected channels are counted + // separately: one channel can report several gaps at once, so a count of entries would + // overstate the number of channels that came back incomplete. let mut gap_reports = 0usize; + let mut channels_with_gaps = 0usize; let started = Instant::now(); for channel in &channels { @@ -88,6 +91,9 @@ fn run(days: u64, only_channel: Option, max_events: Option) { Ok(scan) => { let records = scan.records; gap_reports += scan.gaps.len(); + if !scan.gaps.is_empty() { + channels_with_gaps += 1; + } total += records.len(); if records.len() > widest_channel.1 { widest_channel = (channel.clone(), records.len()); @@ -122,7 +128,8 @@ fn run(days: u64, only_channel: Option, max_events: Option) { println!("days={days}"); println!("channels_scanned={}", channels.len()); println!("channels_failed={failed}"); - println!("channels_with_gaps={gap_reports}"); + println!("channels_with_gaps={channels_with_gaps}"); + println!("gap_entries={gap_reports}"); println!("events={total}"); println!("enumerate_ms={enumerate_ms}"); println!("scan_ms={}", elapsed.as_millis()); From 4b2bbdbb5e42d36feee12ecd9f76ea8e4db7f2da Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 12 Aug 2026 19:34:43 -0400 Subject: [PATCH 85/85] fix(event-log): clear loadError on refresh and pin the failure state refreshLoadedChannels cleared records and coverage gaps but not loadError, so a stale error from an earlier load survived a successful refresh and could hide a later failure. It is now cleared at refresh start, and the failure test asserts the full state: the gap, the loadError message, and that isLoading is false. --- src/workspaces/event-log/evtx-store-coverage.test.ts | 2 ++ src/workspaces/event-log/evtx-store.ts | 1 + 2 files changed, 3 insertions(+) diff --git a/src/workspaces/event-log/evtx-store-coverage.test.ts b/src/workspaces/event-log/evtx-store-coverage.test.ts index 86af2475b..79ee065ad 100644 --- a/src/workspaces/event-log/evtx-store-coverage.test.ts +++ b/src/workspaces/event-log/evtx-store-coverage.test.ts @@ -120,6 +120,8 @@ describe("coverage gaps through the store", () => { state.coverageGaps.some((g) => g.includes("Application") && g.includes("access denied")) ).toBe(true); expect(state.loadError).toContain("Application"); + expect(state.loadError).toContain("access denied"); + expect(state.isLoading).toBe(false); }); }); diff --git a/src/workspaces/event-log/evtx-store.ts b/src/workspaces/event-log/evtx-store.ts index ef5b3b2c3..304a1e13e 100644 --- a/src/workspaces/event-log/evtx-store.ts +++ b/src/workspaces/event-log/evtx-store.ts @@ -387,6 +387,7 @@ export const useEvtxStore = create()((set, get) => ({ loadedChannels: new Set(), selectedRecordId: null, isLoading: true, + loadError: null, loadStartTime: startTime, loadElapsedMs: null, // Cleared with the records they describe. Keeping them would report gaps from a set that is