From 3d2d34c125ee9240e4e4ddf46935104d141578da Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Thu, 29 May 2025 18:47:10 +0000 Subject: [PATCH 01/21] Add skeleton for clp-s design docs --- .../design-clp-structured/background.md | 8 ++++ .../dev-guide/design-clp-structured/index.md | 40 +++++++++++++++++++ .../single-file-archive-format.md | 6 +++ docs/src/dev-guide/index.md | 1 + 4 files changed, 55 insertions(+) create mode 100644 docs/src/dev-guide/design-clp-structured/background.md create mode 100644 docs/src/dev-guide/design-clp-structured/index.md create mode 100644 docs/src/dev-guide/design-clp-structured/single-file-archive-format.md diff --git a/docs/src/dev-guide/design-clp-structured/background.md b/docs/src/dev-guide/design-clp-structured/background.md new file mode 100644 index 0000000000..650630e886 --- /dev/null +++ b/docs/src/dev-guide/design-clp-structured/background.md @@ -0,0 +1,8 @@ +# Background + +The [kv-ir background section](../design-kv-ir-streams/background.md) contains partial background +information about clp-s. + +:::{warning} +🚧 This section is still under construction. +::: diff --git a/docs/src/dev-guide/design-clp-structured/index.md b/docs/src/dev-guide/design-clp-structured/index.md new file mode 100644 index 0000000000..4e9357978a --- /dev/null +++ b/docs/src/dev-guide/design-clp-structured/index.md @@ -0,0 +1,40 @@ +# clp-s + +The compressed-log-processor-structured (abbreviated as clp-s and also known as [μSlope][μSlope]) +archive format is a storage format for large chunks of dynamically structured (e.g., JSON) logs. +Compared to the [KV-IR](../design-kv-ir-streams/index.md) streaming compression format clp-s +achieves higher compression ratios and significantly faster search speeds at the cost of needing to +aggregate log data before compressing the data into archives and requiring memory proportional to +archive size for both compression and search. This makes clp-s unsuitable for real-time or severely +resource-constrained usage but ideal for long-term archival and search. + +::::{grid} 1 1 1 1 +:gutter: 2 + +:::{grid-item-card} +:link: background +Background +^^^ +Any necessary information to understand clp-s' format. +::: + +:::{grid-item-card} +:link: single-file-archive-format +Single-file archive format +^^^ +A detailed reference for the single-file archive format. +::: +:::: + +:::{warning} +🚧 This section is still under construction. +::: + +:::{toctree} +:hidden: + +background +single-file-archive-format +::: + +[μSlope]: https://www.usenix.org/conference/osdi24/presentation/wang-rui diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md new file mode 100644 index 0000000000..c25c70e66f --- /dev/null +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -0,0 +1,6 @@ +# Single-file archive format + +:::{warning} +🚧 This section is still under construction. +::: + diff --git a/docs/src/dev-guide/index.md b/docs/src/dev-guide/index.md index 841a7cf4b4..e6b32601b9 100644 --- a/docs/src/dev-guide/index.md +++ b/docs/src/dev-guide/index.md @@ -81,6 +81,7 @@ tooling-gh-workflows design-project-structure +design-clp-structured/index design-kv-ir-streams/index design-metadata-db design-parsing-wildcard-queries From f06a7b5d4b731321dae933ddc7289f1cf8da6075 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 2 Jun 2025 19:22:11 +0000 Subject: [PATCH 02/21] Fill out basic detail in SFA format section --- .../single-file-archive-format.md | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md index c25c70e66f..f34818ea64 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -1,6 +1,107 @@ # Single-file archive format +The clp-s single-file archive format is designed to offer high compression and fast search on +dynamically structured log data such as JSON logs. This format is optimized for streaming reads in +order to offer high decompression and search performance for archives stored on object storage +systems such as S3. + +This documentation records the details of the single-file archive format and what it enables with +only minimal discussion of design rationale --- for more information about the design decisions +behind clp-s please refer to [our paper on clp-s][μSlope] or our [blog][s3-blog] on optimizing +clp-s for object storage. + +## Format overview + +The single-file archive format is divided into the "header", "metadata", and "files" section as +shown in [Figure 1](#figure-1). + +(figure-1)= +::::{card} +:::{mermaid} +%%{ + init: { + "theme": "base", + "themeVariables": { + "primaryColor": "#0066cc", + "primaryTextColor": "#fff", + "primaryBorderColor": "transparent", + "lineColor": "#007fff", + "secondaryColor": "#007fff", + "tertiaryColor": "#fff" + } + } +}%% +block-beta + columns 1 + header + metadata + block:files_group:1 + columns 1 + files + block:files_detail + mpt["/schema_tree"] sm["/schema_ids"] c["/table_metadata"] vd["/var.dict"] ld["/log.dict"] ad["/array.dict"] t["/0"] + end + end +::: ++++ +**Figure 1**: High-level single-file archive layout. +:::: + +## Header section + +(figure-2)= +::::{card} +:::{mermaid} +%%{ + init: { + "theme": "base", + "themeVariables": { + "primaryColor": "#0066cc", + "primaryTextColor": "#fff", + "primaryBorderColor": "transparent", + "lineColor": "#007fff", + "secondaryColor": "#007fff", + "tertiaryColor": "#fff" + }, + "packet": { + "bitsPerRow": 8, + "bitWidth": 128 + } + } +}%% +packet-beta + 0-3: "Magic Number (0xFD, 0x2F, 0xC5, 0x30)" + 4-5: "Patch Version" + 6: "Minor Version" + 7: "Major Version" + 8-15: "Original Uncompressed Size" + 16-23: "Compressed Archive Size" + 24-55: "Reserved Padding" + 56-59: "Metadata Section Size" + 60-61: "Compression Type" + 62-63: "Reserved Padding" +::: ++++ +**Figure 2**: Layout of the 64-byte archive header. +:::: + +## Metadata section + +### ArchiveInfo packet + +### ArchiveFileInfo packet + +### TimestampDictionary packet + +### RangeIndex packet + +## Files section + +### Merged Parse Tree + :::{warning} 🚧 This section is still under construction. ::: +[s3-blog]: https://blog.yscope.com/optimizing-clp-for-s3-object-storage-b4c502e930ee +[μSlope]: https://www.usenix.org/conference/osdi24/presentation/wang-rui From c31407ea61ed3652df8ef4ba75583c38c00d3f24 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Tue, 3 Jun 2025 17:59:15 +0000 Subject: [PATCH 03/21] Draft header section docs --- .../single-file-archive-format.md | 47 +++++++++++++++++-- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md index f34818ea64..907b41d84f 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -3,17 +3,28 @@ The clp-s single-file archive format is designed to offer high compression and fast search on dynamically structured log data such as JSON logs. This format is optimized for streaming reads in order to offer high decompression and search performance for archives stored on object storage -systems such as S3. +systems such as S3. -This documentation records the details of the single-file archive format and what it enables with -only minimal discussion of design rationale --- for more information about the design decisions +This documentation records the details of the single-file archive v0.3.1 format and what it enables +with only minimal discussion of design rationale --- for more information about the design decisions behind clp-s please refer to [our paper on clp-s][μSlope] or our [blog][s3-blog] on optimizing clp-s for object storage. ## Format overview The single-file archive format is divided into the "header", "metadata", and "files" section as -shown in [Figure 1](#figure-1). +shown in [Figure 1](#figure-1). All archives begin with a 64-byte header that contains important +metadata information such as the archive format version and information needed to read the +"metadata" section. The metadata section is made up of several independent "metadata packets" that +contain archive-level metadata such as the range of timestamp values present in an archive and +information needed to read the "files" section. The files section contains the data structures used +to represent log data and makes up most of the size of an archive; the files section is named as +such because its various components exist as individual files in the multi-file archive format. + +:::{note} +This format is little-endian and all fields in the remainder of this document should be treated as +little-endian unless specified otherwise. +::: (figure-1)= ::::{card} @@ -39,7 +50,7 @@ block-beta columns 1 files block:files_detail - mpt["/schema_tree"] sm["/schema_ids"] c["/table_metadata"] vd["/var.dict"] ld["/log.dict"] ad["/array.dict"] t["/0"] + mpt["/schema_tree"] sm["/schema_ids"] tm["/table_metadata"] vd["/var.dict"] ld["/log.dict"] ad["/array.dict"] t["/0"] end end ::: @@ -49,6 +60,32 @@ block-beta ## Header section +The archive header is a 64-byte unit at the start of a single-file archive containing some of the +most important metadata information about an archive as shown in [Figure 2](#figure-2). The header +begins with a 4-byte magic number which identifies the file as an archive. The magic number is +followed by a 4-byte version number made up of a 2-byte patch version and 1-byte minor and major +version numbers respectively. For version 0.X.Y archives every minor version change is breaking and +readers designed for a given minor version are only sometimes backwards compatible in order to +reduce maintenance while the archive format stabilizes. + +The compressed archive size is the total size of an archive (including the size of the header). The +original uncompressed size field indicates the total size of the data that was compressed into an +archive. The interpretation of "original uncompressed size" changes somewhat based on what data was +ingested into an archive. For example, when ingesting ndjson the field records the number of bytes +of raw JSON data ingested into an archive and when ingesting KV-IR streams the field records the +number of decompressed bytes of KV-IR ingested into an archive (since KV-IR streams are typically +compressed by a general purposed compressor before being stored). + +The metadata section size field indicates the _compressed_ size of the metadata section in bytes. +This field can be used in combination with the uncompressed size field and known header size to +determine the size and offset both the "metadata" and "files" sections. + +The compression type for an archive indicates the general purpose compressor used to compress each +section of the archive and is currently one of: +* 0x0000 - ZStandard + +All "reserved padding" is reserved for use in future versions of the single-file archive format. + (figure-2)= ::::{card} :::{mermaid} From 10e3bd2c700c6e7b543f41e03b402e3bbd0b7b78 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Tue, 3 Jun 2025 18:02:18 +0000 Subject: [PATCH 04/21] Add headers for remainder of subsections --- .../single-file-archive-format.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md index 907b41d84f..37c43f197a 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -134,7 +134,19 @@ packet-beta ## Files section -### Merged Parse Tree +### Merged parse tree + +### Schema map + +### Table metadata + +### Variable dictionary + +### Log-type dictionary + +### Array log-type dictionary + +### Table segments :::{warning} 🚧 This section is still under construction. From 8c0921f589bafbd2dfaf516e0004856519582bfc Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Tue, 3 Jun 2025 20:57:57 +0000 Subject: [PATCH 05/21] Add metadata section preamble --- .../single-file-archive-format.md | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md index 37c43f197a..89859a1b79 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -124,6 +124,29 @@ packet-beta ## Metadata section +The metadata section is made up of a compressed sequence of metadata packets with the following +format: +``` +[num_packets: uint8_t] +[ + [packet_type: uint8_t] + [packet_size: uint32_t] + [packet_content: uint8_t][packet_size] +][num_packets] +``` + +Each metadata packet has a type, size, and arbitrary binary payload. Different packet types make +different choices for encoding content in this binary payload. Since the size of each packet is +well-defined reader implementations can attempt to read archives containing packet types they are +unfamiliar with by simply skipping the corresponding content. Generally this metadata section design +is intended to allow for some degree of forwards-compatibility and extensibility. + +Archives currently support the following metadata packet types, some of which are mandatory: +* 0x00 - ArchiveInfo (mandatory) +* 0x01 - ArchiveFileInfo (mandatory) +* 0x02 - TimestampDictionary +* 0x03 - RangeIndex + ### ArchiveInfo packet ### ArchiveFileInfo packet From e93a22b4011b2e637ff14cd290f9e3f0d1715598 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Wed, 4 Jun 2025 14:21:16 -0400 Subject: [PATCH 06/21] Write up ArchiveInfo and ArchiveFileInfo packets. --- .../single-file-archive-format.md | 96 ++++++++++++++++--- 1 file changed, 84 insertions(+), 12 deletions(-) diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md index 89859a1b79..c904f1b517 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -124,16 +124,28 @@ packet-beta ## Metadata section -The metadata section is made up of a compressed sequence of metadata packets with the following -format: +The metadata section is made up of a compressed sequence of metadata packets with the binary format +shown in [Figure 3](#figure-3). Metadata packets generally contain information that relates to an +entire archive or large sections of an archive. + +(figure-3)= +::::{card} ``` -[num_packets: uint8_t] -[ - [packet_type: uint8_t] - [packet_size: uint32_t] - [packet_content: uint8_t][packet_size] -][num_packets] +MetadataPacket { + packet_type: uint8_t + packet_size: uint32_t + packet_content: uint8_t[packet_size] +} + +MetadataPacketStream { + num_packets: uint8_t + packets: MetadataPacket[num_packets] +} ``` ++++ +**Figure 3**: Metadata section packet stream binary format. The metadata section is equivalent to a +single instance of "MetadataPacketStream". +:::: Each metadata packet has a type, size, and arbitrary binary payload. Different packet types make different choices for encoding content in this binary payload. Since the size of each packet is @@ -142,15 +154,75 @@ unfamiliar with by simply skipping the corresponding content. Generally this met is intended to allow for some degree of forwards-compatibility and extensibility. Archives currently support the following metadata packet types, some of which are mandatory: -* 0x00 - ArchiveInfo (mandatory) -* 0x01 - ArchiveFileInfo (mandatory) -* 0x02 - TimestampDictionary -* 0x03 - RangeIndex +* `0x00` - ArchiveInfo (mandatory) +* `0x01` - ArchiveFileInfo (mandatory) +* `0x02` - TimestampDictionary +* `0x03` - RangeIndex ### ArchiveInfo packet +The ArchiveInfo packet is a msgpack map that currently only records the number of segments in an +archive as shown in [Figure 4](#figure-4). Each archive currently consists of only a single segment +(i.e. there is only one tables segment file), but we still record the number of segments in order +to offer backwards compatibility if we do start splitting tables into multiple segments. + +This metadata packet was originally intended to mimic the "ArchiveMetadata" structure in CLP, but +most of what "ArchiveMetata" records is now present in either the header or the RangeIndex metadata +packet. + +(figure-4)= +::::{card} +```json +{ + "num_segments": 1 +} +``` ++++ +**Figure 4**: Layout of the ArchiveInfo msgpack payload. +:::: + ### ArchiveFileInfo packet +The ArchiveFileInfo packet is a msgpack map that records the name and offset relative to the start +of the files section of each entry in the files section, as shown in [Figure 5](#figure-5). Entries +are ordered by their offset into the files section. This means that the order of entries in the +ArchiveFileInfo packet corresponds to the order in which the components of the files section should +be read to avoid backwards seeks. The offsets stored in this section can be combined with +information from the header to determine the size and absolute offset of every entry in the files +section. + +(figure-5)= +::::{card} +```json +{ + "files": [ + { + "n": "/schema_tree", + "o": 0 + }, + "..." + ] +} +``` ++++ +**Figure 5**: Layout of the ArchiveFileInfo msgpack payload. The files array contains entries for +each file in the files section ordered by their offset. Each file is described by its name "n" and +offset into the files section "o". +:::: + +The ArchiveFileInfo packet has entries entries with the following names in order: +* `"/schema_tree"` - the Merged Parse Tree +* `"/schema_ids"` - the Schema Map +* `"/table_metadata"` - metadata describing the contents of the tables segments +* `"/var.dict"` - the Variable Dictionary +* `"/log.dict"` - the Log-type Dictionary +* `"/array.dict"` - the Array Log-type Dictionary (optional) +* `"/0"` - the first tables segment + +The Array Log-type Dictionary is nominally optional for archives that use structured arrays, but in +practice we currently just store an empty dictionary in this section when structured arrays are +enabled. + ### TimestampDictionary packet ### RangeIndex packet From 727f5b0a228bdb923bb72c31e4905f8224262e61 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Wed, 4 Jun 2025 19:00:49 +0000 Subject: [PATCH 07/21] Add a figure describing the binary format of the timestamp dictionary --- .../single-file-archive-format.md | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md index c904f1b517..de9a11e1cb 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -23,7 +23,9 @@ such because its various components exist as individual files in the multi-file :::{note} This format is little-endian and all fields in the remainder of this document should be treated as -little-endian unless specified otherwise. +little-endian unless specified otherwise. Furthermore, any "struct-like" descriptions of binary +formats should be interpreted as _not_ having padding for alignment, e.g. an int32_t followed by +an int64_t should be interpreted as a 12-byte structure without padding. ::: (figure-1)= @@ -225,6 +227,43 @@ enabled. ### TimestampDictionary packet +(figure-6)= +::::{card} +``` +enum EncodingType : uint64_t { + Epoch = 0x1, + DoubleEpoch = 0x2 +} + +TimestampRangeEntry { + key_len: uint64_t + key: char[key_len] + num_column_ids: uint64_t + column_ids: int32_t[num_column_ids] + encoding_type: EncodingType + epoch_start: union {epochtime_t, double} + epoch_end: union {epochtime_t, double} +} + +TimestampPattern { + pattern_id: uint64_t + pattern_len: uint64_t + pattern: char[pattern_len] +} + +TimestampDictionary { + num_range_entries: uint64_t + range_entries: TimestampRangeEntry[num_entries] + num_patterns: uint64_t + patterns: TimestampPattern[num_patterns] +} + +``` ++++ +**Figure 6**: TimestampDictionary binary payload format. The payload is equivalent to a single +instance of "TimestampDictionary". +:::: + ### RangeIndex packet ## Files section From bd9774e7ce1aa068f24ff3ccc0a5228c3b0cc42e Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Wed, 4 Jun 2025 19:55:56 +0000 Subject: [PATCH 08/21] Add basic write-up for TimestampPattern packet --- .../single-file-archive-format.md | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md index de9a11e1cb..e1a52d61b5 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -84,7 +84,7 @@ determine the size and offset both the "metadata" and "files" sections. The compression type for an archive indicates the general purpose compressor used to compress each section of the archive and is currently one of: -* 0x0000 - ZStandard +* `0x0000` - ZStandard All "reserved padding" is reserved for use in future versions of the single-file archive format. @@ -227,6 +227,12 @@ enabled. ### TimestampDictionary packet +The TimestampDictionary packet is a binary format that records: +1. The key name, corresponding MPT nodes, and range of values for zero or more timestamp columns +2. The format and Id of zero or more timestamp patterns used to encode timestamps in the archive + +The details of the binary format can be seen in [Figure 6](#figure-6). + (figure-6)= ::::{card} ``` @@ -235,7 +241,7 @@ enum EncodingType : uint64_t { DoubleEpoch = 0x2 } -TimestampRangeEntry { +TimestampRange { key_len: uint64_t key: char[key_len] num_column_ids: uint64_t @@ -252,8 +258,8 @@ TimestampPattern { } TimestampDictionary { - num_range_entries: uint64_t - range_entries: TimestampRangeEntry[num_entries] + num_ranges: uint64_t + ranges: TimestampRange[num_entries] num_patterns: uint64_t patterns: TimestampPattern[num_patterns] } @@ -264,6 +270,17 @@ TimestampDictionary { instance of "TimestampDictionary". :::: +The key name in each TimestampRange follows the same +[escaping rules](../../user-guide/reference-json-search-syntax.md) we use for key names in kql +search. Note that we allow each key to map to multiple MPT nodes and have its range recorded as +either integer epoch time or double epoch time in order to handle timestamp columns with polymorphic +types. + +The pattern in each TimestampPattern entry is a format string that follows the specification from +the `clp_s::TimestampPattern` class. The associated pattern_id can be used to uniquely identify each +format string. This allows string timestamps in the archive to be encoded as a tuple of epoch time +and pattern id. + ### RangeIndex packet ## Files section From c6e19fd1efadf0bd50b6f8035e6084c53e459456 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Thu, 5 Jun 2025 17:23:23 +0000 Subject: [PATCH 09/21] Document the archive range index --- .../single-file-archive-format.md | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md index e1a52d61b5..c6ddbf4f67 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -283,6 +283,53 @@ and pattern id. ### RangeIndex packet +The archive range index allows users associate arbitrary properties with each file (or any other +unit of data) ingested into clp-s. Collectively these units of data are referred to as "ingestion +units". Since each archive can potentially aggregate data from multiple files the archive range +index associates these ingestion unit properties with a logical range of records in an archive. +Note that ingestion units can be split across multiple archives in order to maintain a configured +archive size and that in these cases the properties get associated with each chunk of the ingestion +unit in each archive. + +By default we automatically store the following properties about each ingestion unit: +* `"_filename"` - the original filename of the ingestion unit as it was passed to clp-s compression +* `"_file_split_number"` - incremented each time this ingestion-unit is split across antoher archive +* `"_archive_creator_id"` - UUID associated with a particular _invocation_ of compression + +Currently clp-s will store these default properties as well as any properties present in the +metadata section of a KV-IR stream. There will likely be future support for associating arbitrary +additional properties with an ingestion unit at compression time. + +The RangeIndex packet is encoded as a msgpack array with the format shown in [Figure 7](#figure-7). + +(figure-7)= +::::{card} +```json +[ + { + "s": 0, + "e": 100, + "f": { + "_filename": "/my/file.jsonl", + "_file_split_number": 0, + "_archive_creator_id": " 03f2958a-7a2e-448c-a203-60f2cc990d74", + "arbitrary_user_property": "..." + } + }, + "..." +] +``` ++++ +**Figure 7**: Layout of the RangeIndex msgpack payload. Each entry in the array records a start +index "s" and end index "e" indicating that the properties correspond to the logical range of +records [s, e). No entries in the range index have overlapping logical ranges and all entries are +ordered by logical range. The "f" object contains the properties associated with an ingestion unit. +:::: + +Note that properties created and used by clp-s are always prefixed with the "_" character. To avoid +naming collisions with properties created by clp-s users should avoid creating properties with this +prefix. + ## Files section ### Merged parse tree From 975feb15e97e63449ee705cf06db0d439c4761cb Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Thu, 5 Jun 2025 17:33:45 +0000 Subject: [PATCH 10/21] Small edit for consistency --- .../single-file-archive-format.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md index c6ddbf4f67..56f1ecc87f 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -155,11 +155,11 @@ well-defined reader implementations can attempt to read archives containing pack unfamiliar with by simply skipping the corresponding content. Generally this metadata section design is intended to allow for some degree of forwards-compatibility and extensibility. -Archives currently support the following metadata packet types, some of which are mandatory: -* `0x00` - ArchiveInfo (mandatory) -* `0x01` - ArchiveFileInfo (mandatory) -* `0x02` - TimestampDictionary -* `0x03` - RangeIndex +Archives currently support the following metadata packet types, some of which are optional: +* `0x00` - ArchiveInfo +* `0x01` - ArchiveFileInfo +* `0x02` - TimestampDictionary (optional) +* `0x03` - RangeIndex (optional) ### ArchiveInfo packet From abc7a29f12f3b684bf16f37e26ad9a9a31031ff7 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Thu, 5 Jun 2025 17:54:46 +0000 Subject: [PATCH 11/21] Move up Figure 2 in doc --- .../single-file-archive-format.md | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md index 56f1ecc87f..c3617914cf 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -70,24 +70,6 @@ version numbers respectively. For version 0.X.Y archives every minor version cha readers designed for a given minor version are only sometimes backwards compatible in order to reduce maintenance while the archive format stabilizes. -The compressed archive size is the total size of an archive (including the size of the header). The -original uncompressed size field indicates the total size of the data that was compressed into an -archive. The interpretation of "original uncompressed size" changes somewhat based on what data was -ingested into an archive. For example, when ingesting ndjson the field records the number of bytes -of raw JSON data ingested into an archive and when ingesting KV-IR streams the field records the -number of decompressed bytes of KV-IR ingested into an archive (since KV-IR streams are typically -compressed by a general purposed compressor before being stored). - -The metadata section size field indicates the _compressed_ size of the metadata section in bytes. -This field can be used in combination with the uncompressed size field and known header size to -determine the size and offset both the "metadata" and "files" sections. - -The compression type for an archive indicates the general purpose compressor used to compress each -section of the archive and is currently one of: -* `0x0000` - ZStandard - -All "reserved padding" is reserved for use in future versions of the single-file archive format. - (figure-2)= ::::{card} :::{mermaid} @@ -124,6 +106,24 @@ packet-beta **Figure 2**: Layout of the 64-byte archive header. :::: +The compressed archive size is the total size of an archive (including the size of the header). The +original uncompressed size field indicates the total size of the data that was compressed into an +archive. The interpretation of "original uncompressed size" changes somewhat based on what data was +ingested into an archive. For example, when ingesting ndjson the field records the number of bytes +of raw JSON data ingested into an archive and when ingesting KV-IR streams the field records the +number of decompressed bytes of KV-IR ingested into an archive (since KV-IR streams are typically +compressed by a general purposed compressor before being stored). + +The metadata section size field indicates the _compressed_ size of the metadata section in bytes. +This field can be used in combination with the uncompressed size field and known header size to +determine the size and offset both the "metadata" and "files" sections. + +The compression type for an archive indicates the general purpose compressor used to compress each +section of the archive and is currently one of: +* `0x0000` - ZStandard + +All "reserved padding" is reserved for use in future versions of the single-file archive format. + ## Metadata section The metadata section is made up of a compressed sequence of metadata packets with the binary format From 3229a3f2a5ebd0e92eeb44e9a6f6372db0c26970 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Thu, 5 Jun 2025 18:02:46 +0000 Subject: [PATCH 12/21] Address some rabbit comments --- .../single-file-archive-format.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md index c3617914cf..34a9cfd484 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -2,12 +2,11 @@ The clp-s single-file archive format is designed to offer high compression and fast search on dynamically structured log data such as JSON logs. This format is optimized for streaming reads in -order to offer high decompression and search performance for archives stored on object storage -systems such as S3. +order to enable high performance for archives stored on object storage systems such as S3. This documentation records the details of the single-file archive v0.3.1 format and what it enables with only minimal discussion of design rationale --- for more information about the design decisions -behind clp-s please refer to [our paper on clp-s][μSlope] or our [blog][s3-blog] on optimizing +behind clp-s, please refer to [our paper on clp-s][μSlope], or our [blog][s3-blog] on optimizing clp-s for object storage. ## Format overview @@ -67,8 +66,8 @@ most important metadata information about an archive as shown in [Figure 2](#fig begins with a 4-byte magic number which identifies the file as an archive. The magic number is followed by a 4-byte version number made up of a 2-byte patch version and 1-byte minor and major version numbers respectively. For version 0.X.Y archives every minor version change is breaking and -readers designed for a given minor version are only sometimes backwards compatible in order to -reduce maintenance while the archive format stabilizes. +readers designed for a given minor version are only sometimes backwards compatible to reduce +maintenance while the archive format stabilizes. (figure-2)= ::::{card} @@ -212,7 +211,7 @@ each file in the files section ordered by their offset. Each file is described b offset into the files section "o". :::: -The ArchiveFileInfo packet has entries entries with the following names in order: +The ArchiveFileInfo packet has entries with the following names in order: * `"/schema_tree"` - the Merged Parse Tree * `"/schema_ids"` - the Schema Map * `"/table_metadata"` - metadata describing the contents of the tables segments @@ -273,8 +272,7 @@ instance of "TimestampDictionary". The key name in each TimestampRange follows the same [escaping rules](../../user-guide/reference-json-search-syntax.md) we use for key names in kql search. Note that we allow each key to map to multiple MPT nodes and have its range recorded as -either integer epoch time or double epoch time in order to handle timestamp columns with polymorphic -types. +either integer epoch time or double epoch time to handle timestamp columns with polymorphic types. The pattern in each TimestampPattern entry is a format string that follows the specification from the `clp_s::TimestampPattern` class. The associated pattern_id can be used to uniquely identify each From bd2a7052b5c1959bc3a5d264c37180a323b56b38 Mon Sep 17 00:00:00 2001 From: Devin Gibson Date: Mon, 16 Jun 2025 15:06:40 -0400 Subject: [PATCH 13/21] Apply suggestions from code review Co-authored-by: quinntaylormitchell --- .../design-clp-structured/background.md | 2 +- .../dev-guide/design-clp-structured/index.md | 8 +- .../single-file-archive-format.md | 81 ++++++++++--------- 3 files changed, 46 insertions(+), 45 deletions(-) diff --git a/docs/src/dev-guide/design-clp-structured/background.md b/docs/src/dev-guide/design-clp-structured/background.md index 650630e886..2bd99bc62f 100644 --- a/docs/src/dev-guide/design-clp-structured/background.md +++ b/docs/src/dev-guide/design-clp-structured/background.md @@ -1,6 +1,6 @@ # Background -The [kv-ir background section](../design-kv-ir-streams/background.md) contains partial background +The [KV-IR background section](../design-kv-ir-streams/background.md) contains partial background information about clp-s. :::{warning} diff --git a/docs/src/dev-guide/design-clp-structured/index.md b/docs/src/dev-guide/design-clp-structured/index.md index 4e9357978a..9fa72c16cc 100644 --- a/docs/src/dev-guide/design-clp-structured/index.md +++ b/docs/src/dev-guide/design-clp-structured/index.md @@ -1,8 +1,8 @@ # clp-s -The compressed-log-processor-structured (abbreviated as clp-s and also known as [μSlope][μSlope]) -archive format is a storage format for large chunks of dynamically structured (e.g., JSON) logs. -Compared to the [KV-IR](../design-kv-ir-streams/index.md) streaming compression format clp-s +The compressed-log-processor-structured (abbreviated `clp-s`, also known as [μSlope][μSlope]) +archive format is a storage format for large chunks of dynamically-structured (e.g., JSON) logs. +Compared to the [KV-IR](../design-kv-ir-streams/index.md) streaming compression format, `clp-s` achieves higher compression ratios and significantly faster search speeds at the cost of needing to aggregate log data before compressing the data into archives and requiring memory proportional to archive size for both compression and search. This makes clp-s unsuitable for real-time or severely @@ -15,7 +15,7 @@ resource-constrained usage but ideal for long-term archival and search. :link: background Background ^^^ -Any necessary information to understand clp-s' format. +All necessary information needed to understand `clp-s`' format. ::: :::{grid-item-card} diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md index 34a9cfd484..007ef5ba1a 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -1,30 +1,30 @@ # Single-file archive format -The clp-s single-file archive format is designed to offer high compression and fast search on -dynamically structured log data such as JSON logs. This format is optimized for streaming reads in -order to enable high performance for archives stored on object storage systems such as S3. +The `clp-s` single-file archive format is designed to offer high compression and fast search on +dynamically-structured log data such as JSON logs. This format is optimized for streaming reads in +order to enable high performance for archives stored on object storage systems such as [S3][amazon-s3]. This documentation records the details of the single-file archive v0.3.1 format and what it enables -with only minimal discussion of design rationale --- for more information about the design decisions -behind clp-s, please refer to [our paper on clp-s][μSlope], or our [blog][s3-blog] on optimizing -clp-s for object storage. +with minimal discussion of design rationale. For more information about the design decisions +behind `clp-s`, please refer to [our paper on `clp-s`][μSlope], or our [blog][s3-blog] on optimizing +`clp-s` for object storage. ## Format overview -The single-file archive format is divided into the "header", "metadata", and "files" section as +The single-file archive format is divided into the `header`, `metadata`, and `files` sections, as shown in [Figure 1](#figure-1). All archives begin with a 64-byte header that contains important -metadata information such as the archive format version and information needed to read the -"metadata" section. The metadata section is made up of several independent "metadata packets" that +metadata, such as the archive format version and information needed to read the +`metadata` section. The `metadata` section is made up of several independent "metadata packets" that contain archive-level metadata such as the range of timestamp values present in an archive and -information needed to read the "files" section. The files section contains the data structures used -to represent log data and makes up most of the size of an archive; the files section is named as +information needed to read the `files` section. The `files` section contains the data structures used +to represent log data and makes up most of the size of an archive; the `files` section is named as such because its various components exist as individual files in the multi-file archive format. :::{note} -This format is little-endian and all fields in the remainder of this document should be treated as +The single-file archive format is little-endian. Therefore, all fields in the remainder of this document should be treated as little-endian unless specified otherwise. Furthermore, any "struct-like" descriptions of binary -formats should be interpreted as _not_ having padding for alignment, e.g. an int32_t followed by -an int64_t should be interpreted as a 12-byte structure without padding. +formats should be interpreted as _not_ having padding for alignment, e.g. an `int32_t` followed by +an `int64_t` should be interpreted as a 12-byte structure without padding. ::: (figure-1)= @@ -62,12 +62,12 @@ block-beta ## Header section The archive header is a 64-byte unit at the start of a single-file archive containing some of the -most important metadata information about an archive as shown in [Figure 2](#figure-2). The header +most important metadata information about an archive, as shown in [Figure 2](#figure-2). The header begins with a 4-byte magic number which identifies the file as an archive. The magic number is followed by a 4-byte version number made up of a 2-byte patch version and 1-byte minor and major version numbers respectively. For version 0.X.Y archives every minor version change is breaking and -readers designed for a given minor version are only sometimes backwards compatible to reduce -maintenance while the archive format stabilizes. +to reduce maintenance while the archive format stabilizes, +not all readers designed for a given minor version are backwards-compatible. (figure-2)= ::::{card} @@ -117,7 +117,7 @@ The metadata section size field indicates the _compressed_ size of the metadata This field can be used in combination with the uncompressed size field and known header size to determine the size and offset both the "metadata" and "files" sections. -The compression type for an archive indicates the general purpose compressor used to compress each +The compression type for an archive indicates the general-purpose compressor used to compress each section of the archive and is currently one of: * `0x0000` - ZStandard @@ -163,12 +163,12 @@ Archives currently support the following metadata packet types, some of which ar ### ArchiveInfo packet The ArchiveInfo packet is a msgpack map that currently only records the number of segments in an -archive as shown in [Figure 4](#figure-4). Each archive currently consists of only a single segment +archive, as shown in [Figure 4](#figure-4). Each archive currently consists of only a single segment (i.e. there is only one tables segment file), but we still record the number of segments in order to offer backwards compatibility if we do start splitting tables into multiple segments. This metadata packet was originally intended to mimic the "ArchiveMetadata" structure in CLP, but -most of what "ArchiveMetata" records is now present in either the header or the RangeIndex metadata +most of what "ArchiveMetadata" records is now present in either the header or the RangeIndex metadata packet. (figure-4)= @@ -185,11 +185,11 @@ packet. ### ArchiveFileInfo packet The ArchiveFileInfo packet is a msgpack map that records the name and offset relative to the start -of the files section of each entry in the files section, as shown in [Figure 5](#figure-5). Entries -are ordered by their offset into the files section. This means that the order of entries in the -ArchiveFileInfo packet corresponds to the order in which the components of the files section should +of the `files` section of each entry in that section, as shown in [Figure 5](#figure-5). Entries +are ordered by their offset into the `files` section. This means that the order of entries in the +ArchiveFileInfo packet corresponds to the order in which the components of the `files` section should be read to avoid backwards seeks. The offsets stored in this section can be combined with -information from the header to determine the size and absolute offset of every entry in the files +information from the header to determine the size and absolute offset of every entry in the `files` section. (figure-5)= @@ -206,9 +206,9 @@ section. } ``` +++ -**Figure 5**: Layout of the ArchiveFileInfo msgpack payload. The files array contains entries for -each file in the files section ordered by their offset. Each file is described by its name "n" and -offset into the files section "o". +**Figure 5**: Layout of the ArchiveFileInfo msgpack payload. The `"files"` array contains entries for +each file in the `files` section, ordered by their offset. Each file is described by its name `"n"` and +offset into the `files` section `"o"`. :::: The ArchiveFileInfo packet has entries with the following names in order: @@ -228,7 +228,7 @@ enabled. The TimestampDictionary packet is a binary format that records: 1. The key name, corresponding MPT nodes, and range of values for zero or more timestamp columns -2. The format and Id of zero or more timestamp patterns used to encode timestamps in the archive +2. The format and ID of zero or more timestamp patterns used to encode timestamps in the archive The details of the binary format can be seen in [Figure 6](#figure-6). @@ -270,8 +270,8 @@ instance of "TimestampDictionary". :::: The key name in each TimestampRange follows the same -[escaping rules](../../user-guide/reference-json-search-syntax.md) we use for key names in kql -search. Note that we allow each key to map to multiple MPT nodes and have its range recorded as +[escaping rules](../../user-guide/reference-json-search-syntax.md) we use for key names in KQL +search. Note that we allow each key to map to multiple MPT nodes, and that each key's range can be recorded as either integer epoch time or double epoch time to handle timestamp columns with polymorphic types. The pattern in each TimestampPattern entry is a format string that follows the specification from @@ -281,17 +281,17 @@ and pattern id. ### RangeIndex packet -The archive range index allows users associate arbitrary properties with each file (or any other -unit of data) ingested into clp-s. Collectively these units of data are referred to as "ingestion +The archive range index allows users to associate arbitrary properties with each file (or any other +unit of data) ingested into `clp-s`. Collectively, these units of data are referred to as "ingestion units". Since each archive can potentially aggregate data from multiple files the archive range index associates these ingestion unit properties with a logical range of records in an archive. Note that ingestion units can be split across multiple archives in order to maintain a configured -archive size and that in these cases the properties get associated with each chunk of the ingestion +archive size, and that in these cases the properties become associated with each chunk of the ingestion unit in each archive. By default we automatically store the following properties about each ingestion unit: -* `"_filename"` - the original filename of the ingestion unit as it was passed to clp-s compression -* `"_file_split_number"` - incremented each time this ingestion-unit is split across antoher archive +* `"_filename"` - the original filename of the ingestion unit as it was passed to `clp-s` during compression +* `"_file_split_number"` - incremented each time this ingestion-unit is split across another archive * `"_archive_creator_id"` - UUID associated with a particular _invocation_ of compression Currently clp-s will store these default properties as well as any properties present in the @@ -319,13 +319,13 @@ The RangeIndex packet is encoded as a msgpack array with the format shown in [Fi ``` +++ **Figure 7**: Layout of the RangeIndex msgpack payload. Each entry in the array records a start -index "s" and end index "e" indicating that the properties correspond to the logical range of -records [s, e). No entries in the range index have overlapping logical ranges and all entries are -ordered by logical range. The "f" object contains the properties associated with an ingestion unit. +index `"s"` and end index `"e"` indicating that the properties correspond to the logical range of +records `[s, e)`. No entries in the range index have overlapping logical ranges, and all entries are +ordered by logical range. The `"f"` object contains the properties associated with an ingestion unit. :::: -Note that properties created and used by clp-s are always prefixed with the "_" character. To avoid -naming collisions with properties created by clp-s users should avoid creating properties with this +Note that properties created and used by `clp-s` are always prefixed with the `_` character. To avoid +naming collisions with properties created by `clp-s`, users should avoid creating properties with this prefix. ## Files section @@ -350,3 +350,4 @@ prefix. [s3-blog]: https://blog.yscope.com/optimizing-clp-for-s3-object-storage-b4c502e930ee [μSlope]: https://www.usenix.org/conference/osdi24/presentation/wang-rui +[amazon-s3]: https://aws.amazon.com/s3/ From 3986176d9673228090891016797c32dc0538fafd Mon Sep 17 00:00:00 2001 From: Devin Gibson Date: Mon, 16 Jun 2025 15:18:12 -0400 Subject: [PATCH 14/21] Apply suggestions from code review Co-authored-by: quinntaylormitchell --- .../dev-guide/design-clp-structured/index.md | 8 ++++---- .../single-file-archive-format.md | 17 ++++++++--------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/docs/src/dev-guide/design-clp-structured/index.md b/docs/src/dev-guide/design-clp-structured/index.md index 9fa72c16cc..ee3806a216 100644 --- a/docs/src/dev-guide/design-clp-structured/index.md +++ b/docs/src/dev-guide/design-clp-structured/index.md @@ -3,10 +3,10 @@ The compressed-log-processor-structured (abbreviated `clp-s`, also known as [μSlope][μSlope]) archive format is a storage format for large chunks of dynamically-structured (e.g., JSON) logs. Compared to the [KV-IR](../design-kv-ir-streams/index.md) streaming compression format, `clp-s` -achieves higher compression ratios and significantly faster search speeds at the cost of needing to -aggregate log data before compressing the data into archives and requiring memory proportional to -archive size for both compression and search. This makes clp-s unsuitable for real-time or severely -resource-constrained usage but ideal for long-term archival and search. +achieves higher compression ratios and significantly faster search speeds. To accomplish this, `clp-s` needs to +aggregate log data before compressing the data into archives. In addition, it requires memory proportional to +archive size for both compression and search. These constraints make `clp-s` unsuitable for real-time or severely +resource-constrained usage, but ideal for long-term archival and search. ::::{grid} 1 1 1 1 :gutter: 2 diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md index 007ef5ba1a..4896fdae41 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -61,10 +61,10 @@ block-beta ## Header section -The archive header is a 64-byte unit at the start of a single-file archive containing some of the +The archive header is a 64-byte unit at the start of a single-file archive that contains some of the most important metadata information about an archive, as shown in [Figure 2](#figure-2). The header begins with a 4-byte magic number which identifies the file as an archive. The magic number is -followed by a 4-byte version number made up of a 2-byte patch version and 1-byte minor and major +followed by a 4-byte version number, comprised of a 2-byte patch version number, followed by 1-byte minor and major version numbers respectively. For version 0.X.Y archives every minor version change is breaking and to reduce maintenance while the archive format stabilizes, not all readers designed for a given minor version are backwards-compatible. @@ -144,8 +144,8 @@ MetadataPacketStream { } ``` +++ -**Figure 3**: Metadata section packet stream binary format. The metadata section is equivalent to a -single instance of "MetadataPacketStream". +**Figure 3**: `Metadata` section packet stream binary format. The `metadata` section is equivalent to a +single instance of `MetadataPacketStream`. :::: Each metadata packet has a type, size, and arbitrary binary payload. Different packet types make @@ -221,8 +221,7 @@ The ArchiveFileInfo packet has entries with the following names in order: * `"/0"` - the first tables segment The Array Log-type Dictionary is nominally optional for archives that use structured arrays, but in -practice we currently just store an empty dictionary in this section when structured arrays are -enabled. +practice, when structured arrays are enabled, we just store an empty dictionary in this section. ### TimestampDictionary packet @@ -274,8 +273,8 @@ The key name in each TimestampRange follows the same search. Note that we allow each key to map to multiple MPT nodes, and that each key's range can be recorded as either integer epoch time or double epoch time to handle timestamp columns with polymorphic types. -The pattern in each TimestampPattern entry is a format string that follows the specification from -the `clp_s::TimestampPattern` class. The associated pattern_id can be used to uniquely identify each +The `pattern` in each TimestampPattern entry is a format string that follows the specification from +the `clp_s::TimestampPattern` class. The associated `pattern_id` can be used to uniquely identify each format string. This allows string timestamps in the archive to be encoded as a tuple of epoch time and pattern id. @@ -283,7 +282,7 @@ and pattern id. The archive range index allows users to associate arbitrary properties with each file (or any other unit of data) ingested into `clp-s`. Collectively, these units of data are referred to as "ingestion -units". Since each archive can potentially aggregate data from multiple files the archive range +units". Since each archive can potentially aggregate data from multiple files, the archive range index associates these ingestion unit properties with a logical range of records in an archive. Note that ingestion units can be split across multiple archives in order to maintain a configured archive size, and that in these cases the properties become associated with each chunk of the ingestion From ffd31bba4d5e190fcbd47149a3c79d6c77fd211f Mon Sep 17 00:00:00 2001 From: Devin Gibson Date: Mon, 16 Jun 2025 15:51:40 -0400 Subject: [PATCH 15/21] Apply suggestions from code review Co-authored-by: quinntaylormitchell --- .../single-file-archive-format.md | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md index 4896fdae41..690952b111 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -105,17 +105,17 @@ packet-beta **Figure 2**: Layout of the 64-byte archive header. :::: -The compressed archive size is the total size of an archive (including the size of the header). The -original uncompressed size field indicates the total size of the data that was compressed into an -archive. The interpretation of "original uncompressed size" changes somewhat based on what data was -ingested into an archive. For example, when ingesting ndjson the field records the number of bytes -of raw JSON data ingested into an archive and when ingesting KV-IR streams the field records the -number of decompressed bytes of KV-IR ingested into an archive (since KV-IR streams are typically +The value in `compressed archive size` indicates the total size of the archive, including the size of the header. The +`original uncompressed size` field indicates the total size of the data that was compressed into the +archive. The interpretation of `original uncompressed size` changes somewhat based on what data was +ingested into the archive. For example, when ingesting `ndjson`, the field records the number of bytes +of raw JSON data ingested into the archive --- however, when ingesting KV-IR streams, the field records the +number of decompressed bytes of KV-IR ingested into the archive (since KV-IR streams are typically compressed by a general purposed compressor before being stored). -The metadata section size field indicates the _compressed_ size of the metadata section in bytes. -This field can be used in combination with the uncompressed size field and known header size to -determine the size and offset both the "metadata" and "files" sections. +The `metadata section size` field indicates the _compressed_ size of the metadata section in bytes. +This field can be used in combination with the `original uncompressed size` field and known header size to +determine the size and offset of both the `metadata` and `files` sections. The compression type for an archive indicates the general-purpose compressor used to compress each section of the archive and is currently one of: @@ -126,8 +126,8 @@ All "reserved padding" is reserved for use in future versions of the single-file ## Metadata section The metadata section is made up of a compressed sequence of metadata packets with the binary format -shown in [Figure 3](#figure-3). Metadata packets generally contain information that relates to an -entire archive or large sections of an archive. +shown in [Figure 3](#figure-3). Metadata packets generally contain information that relates to the +entire archive or large sections of the archive. (figure-3)= ::::{card} From 509c448e3eff3711191aea5adc431ac7e05b7665 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 16 Jun 2025 19:52:23 +0000 Subject: [PATCH 16/21] Address some more review comments --- .../single-file-archive-format.md | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md index 690952b111..264bdf7475 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -2,12 +2,13 @@ The `clp-s` single-file archive format is designed to offer high compression and fast search on dynamically-structured log data such as JSON logs. This format is optimized for streaming reads in -order to enable high performance for archives stored on object storage systems such as [S3][amazon-s3]. +order to enable high performance for archives stored on object storage systems such as +[S3][amazon-s3]. -This documentation records the details of the single-file archive v0.3.1 format and what it enables -with minimal discussion of design rationale. For more information about the design decisions -behind `clp-s`, please refer to [our paper on `clp-s`][μSlope], or our [blog][s3-blog] on optimizing -`clp-s` for object storage. +This documentation records the details of the single-file archive (v0.3.1) format and what it +enables, with minimal discussion of design rationale. For more information about the design +decisions behind `clp-s`, please refer to [our paper on `clp-s`][μSlope], or our [blog][s3-blog] on +optimizing `clp-s` for object storage. ## Format overview @@ -16,15 +17,16 @@ shown in [Figure 1](#figure-1). All archives begin with a 64-byte header that co metadata, such as the archive format version and information needed to read the `metadata` section. The `metadata` section is made up of several independent "metadata packets" that contain archive-level metadata such as the range of timestamp values present in an archive and -information needed to read the `files` section. The `files` section contains the data structures used -to represent log data and makes up most of the size of an archive; the `files` section is named as -such because its various components exist as individual files in the multi-file archive format. +information needed to read the `files` section. The `files` section contains the data structures +used to represent log data and makes up most of the size of an archive; the `files` section is named +as such because its various components exist as individual files in the multi-file archive format. :::{note} -The single-file archive format is little-endian. Therefore, all fields in the remainder of this document should be treated as -little-endian unless specified otherwise. Furthermore, any "struct-like" descriptions of binary -formats should be interpreted as _not_ having padding for alignment, e.g. an `int32_t` followed by -an `int64_t` should be interpreted as a 12-byte structure without padding. +The single-file archive format is little-endian. Therefore, all fields in the remainder of this +document should be treated as little-endian unless specified otherwise. Furthermore, any +"struct-like" descriptions of binary formats should be interpreted as _not_ having padding for +alignment, e.g. an `int32_t` followed by an `int64_t` should be interpreted as a 12-byte structure +without padding. ::: (figure-1)= @@ -64,10 +66,10 @@ block-beta The archive header is a 64-byte unit at the start of a single-file archive that contains some of the most important metadata information about an archive, as shown in [Figure 2](#figure-2). The header begins with a 4-byte magic number which identifies the file as an archive. The magic number is -followed by a 4-byte version number, comprised of a 2-byte patch version number, followed by 1-byte minor and major -version numbers respectively. For version 0.X.Y archives every minor version change is breaking and -to reduce maintenance while the archive format stabilizes, -not all readers designed for a given minor version are backwards-compatible. +followed by a 4-byte version number, comprised of a 2-byte patch version number, followed by 1-byte +minor and major version numbers respectively. For version 0.X.Y archives, every minor version +change is breaking. To reduce maintenance while the archive format stabilizes, not all readers +designed for a given minor version are backwards-compatible. (figure-2)= ::::{card} From 231bda4103636e770492b309357609cfb63ff565 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 16 Jun 2025 20:03:46 +0000 Subject: [PATCH 17/21] Address more review comments --- .../single-file-archive-format.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md index 264bdf7475..f7aa15a4d0 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -123,13 +123,14 @@ The compression type for an archive indicates the general-purpose compressor use section of the archive and is currently one of: * `0x0000` - ZStandard -All "reserved padding" is reserved for use in future versions of the single-file archive format. +All `reserved padding` fields are reserved for use in future versions of the single-file archive +format. ## Metadata section -The metadata section is made up of a compressed sequence of metadata packets with the binary format -shown in [Figure 3](#figure-3). Metadata packets generally contain information that relates to the -entire archive or large sections of the archive. +The `metadata`` section is made up of a compressed sequence of metadata packets encoded with the +binary format shown in [Figure 3](#figure-3). Metadata packets generally contain information that +relates to the entire archive or large sections of the archive. (figure-3)= ::::{card} @@ -295,9 +296,9 @@ By default we automatically store the following properties about each ingestion * `"_file_split_number"` - incremented each time this ingestion-unit is split across another archive * `"_archive_creator_id"` - UUID associated with a particular _invocation_ of compression -Currently clp-s will store these default properties as well as any properties present in the -metadata section of a KV-IR stream. There will likely be future support for associating arbitrary -additional properties with an ingestion unit at compression time. +Currently `clp-s` will associate these default properties as well as any properties present in the +metadata section of a KV-IR stream with each ingestion unit. There will likely be future support +for associating arbitrary additional properties with an ingestion unit at compression time. The RangeIndex packet is encoded as a msgpack array with the format shown in [Figure 7](#figure-7). From 6b0c8fcddfcbddfba66ed297a3c5d2591a57197f Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 16 Jun 2025 20:10:58 +0000 Subject: [PATCH 18/21] Fix sfa docs file formatting --- .../single-file-archive-format.md | 66 ++++++++++--------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md index f7aa15a4d0..e27fe30673 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -53,7 +53,8 @@ block-beta columns 1 files block:files_detail - mpt["/schema_tree"] sm["/schema_ids"] tm["/table_metadata"] vd["/var.dict"] ld["/log.dict"] ad["/array.dict"] t["/0"] + mpt["/schema_tree"] sm["/schema_ids"] tm["/table_metadata"] vd["/var.dict"] ld["/log.dict"] + ad["/array.dict"] t["/0"] end end ::: @@ -107,17 +108,17 @@ packet-beta **Figure 2**: Layout of the 64-byte archive header. :::: -The value in `compressed archive size` indicates the total size of the archive, including the size of the header. The -`original uncompressed size` field indicates the total size of the data that was compressed into the -archive. The interpretation of `original uncompressed size` changes somewhat based on what data was -ingested into the archive. For example, when ingesting `ndjson`, the field records the number of bytes -of raw JSON data ingested into the archive --- however, when ingesting KV-IR streams, the field records the -number of decompressed bytes of KV-IR ingested into the archive (since KV-IR streams are typically -compressed by a general purposed compressor before being stored). +The value in `compressed archive size` indicates the total size of the archive, including the size +of the header. The `original uncompressed size` field indicates the total size of the data that was +compressed into the archive. The interpretation of `original uncompressed size` changes somewhat +based on what data was ingested into the archive. For example, when ingesting `ndjson`, the field +records the number of bytes of raw JSON data ingested into the archive --- however, when ingesting +KV-IR streams, the field records the number of decompressed bytes of KV-IR ingested into the archive +(since KV-IR streams are typically compressed by a general purposed compressor before being stored). The `metadata section size` field indicates the _compressed_ size of the metadata section in bytes. -This field can be used in combination with the `original uncompressed size` field and known header size to -determine the size and offset of both the `metadata` and `files` sections. +This field can be used in combination with the `original uncompressed size` field and known header +size to determine the size and offset of both the `metadata` and `files` sections. The compression type for an archive indicates the general-purpose compressor used to compress each section of the archive and is currently one of: @@ -147,8 +148,8 @@ MetadataPacketStream { } ``` +++ -**Figure 3**: `Metadata` section packet stream binary format. The `metadata` section is equivalent to a -single instance of `MetadataPacketStream`. +**Figure 3**: The `metadata` section packet stream binary format. The `metadata` section is +equivalent to a single instance of `MetadataPacketStream`. :::: Each metadata packet has a type, size, and arbitrary binary payload. Different packet types make @@ -171,8 +172,8 @@ archive, as shown in [Figure 4](#figure-4). Each archive currently consists of o to offer backwards compatibility if we do start splitting tables into multiple segments. This metadata packet was originally intended to mimic the "ArchiveMetadata" structure in CLP, but -most of what "ArchiveMetadata" records is now present in either the header or the RangeIndex metadata -packet. +most of what "ArchiveMetadata" records is now present in either the header or the RangeIndex +metadata packet. (figure-4)= ::::{card} @@ -190,8 +191,8 @@ packet. The ArchiveFileInfo packet is a msgpack map that records the name and offset relative to the start of the `files` section of each entry in that section, as shown in [Figure 5](#figure-5). Entries are ordered by their offset into the `files` section. This means that the order of entries in the -ArchiveFileInfo packet corresponds to the order in which the components of the `files` section should -be read to avoid backwards seeks. The offsets stored in this section can be combined with +ArchiveFileInfo packet corresponds to the order in which the components of the `files` section +should be read to avoid backwards seeks. The offsets stored in this section can be combined with information from the header to determine the size and absolute offset of every entry in the `files` section. @@ -209,9 +210,9 @@ section. } ``` +++ -**Figure 5**: Layout of the ArchiveFileInfo msgpack payload. The `"files"` array contains entries for -each file in the `files` section, ordered by their offset. Each file is described by its name `"n"` and -offset into the `files` section `"o"`. +**Figure 5**: Layout of the ArchiveFileInfo msgpack payload. The `"files"` array contains entries +for each file in the `files` section, ordered by their offset. Each file is described by its name +`"n"` and offset into the `files` section `"o"`. :::: The ArchiveFileInfo packet has entries with the following names in order: @@ -273,13 +274,14 @@ instance of "TimestampDictionary". The key name in each TimestampRange follows the same [escaping rules](../../user-guide/reference-json-search-syntax.md) we use for key names in KQL -search. Note that we allow each key to map to multiple MPT nodes, and that each key's range can be recorded as -either integer epoch time or double epoch time to handle timestamp columns with polymorphic types. +search. Note that we allow each key to map to multiple MPT nodes, and that each key's range can be +recorded as either integer epoch time or double epoch time to handle timestamp columns with +polymorphic types. The `pattern` in each TimestampPattern entry is a format string that follows the specification from -the `clp_s::TimestampPattern` class. The associated `pattern_id` can be used to uniquely identify each -format string. This allows string timestamps in the archive to be encoded as a tuple of epoch time -and pattern id. +the `clp_s::TimestampPattern` class. The associated `pattern_id` can be used to uniquely identify +each format string. This allows string timestamps in the archive to be encoded as a tuple of epoch +time and pattern id. ### RangeIndex packet @@ -288,11 +290,12 @@ unit of data) ingested into `clp-s`. Collectively, these units of data are refer units". Since each archive can potentially aggregate data from multiple files, the archive range index associates these ingestion unit properties with a logical range of records in an archive. Note that ingestion units can be split across multiple archives in order to maintain a configured -archive size, and that in these cases the properties become associated with each chunk of the ingestion -unit in each archive. +archive size, and that in these cases the properties become associated with each chunk of the +ingestion unit in each archive. By default we automatically store the following properties about each ingestion unit: -* `"_filename"` - the original filename of the ingestion unit as it was passed to `clp-s` during compression +* `"_filename"` - the original filename of the ingestion unit as it was passed to `clp-s` during +compression * `"_file_split_number"` - incremented each time this ingestion-unit is split across another archive * `"_archive_creator_id"` - UUID associated with a particular _invocation_ of compression @@ -323,12 +326,13 @@ The RangeIndex packet is encoded as a msgpack array with the format shown in [Fi **Figure 7**: Layout of the RangeIndex msgpack payload. Each entry in the array records a start index `"s"` and end index `"e"` indicating that the properties correspond to the logical range of records `[s, e)`. No entries in the range index have overlapping logical ranges, and all entries are -ordered by logical range. The `"f"` object contains the properties associated with an ingestion unit. +ordered by logical range. The `"f"` object contains the properties associated with an ingestion +unit. :::: -Note that properties created and used by `clp-s` are always prefixed with the `_` character. To avoid -naming collisions with properties created by `clp-s`, users should avoid creating properties with this -prefix. +Note that properties created and used by `clp-s` are always prefixed with the `_` character. To +avoid naming collisions with properties created by `clp-s`, users should avoid creating properties +with this prefix. ## Files section From 3849efe95c76f4e1237ae45d1750240eb2fcdb31 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 16 Jun 2025 20:18:56 +0000 Subject: [PATCH 19/21] Rewrite header format with struct-like description --- .../single-file-archive-format.md | 58 +++++++------------ 1 file changed, 21 insertions(+), 37 deletions(-) diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md index e27fe30673..8c6384c648 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -74,62 +74,46 @@ designed for a given minor version are backwards-compatible. (figure-2)= ::::{card} -:::{mermaid} -%%{ - init: { - "theme": "base", - "themeVariables": { - "primaryColor": "#0066cc", - "primaryTextColor": "#fff", - "primaryBorderColor": "transparent", - "lineColor": "#007fff", - "secondaryColor": "#007fff", - "tertiaryColor": "#fff" - }, - "packet": { - "bitsPerRow": 8, - "bitWidth": 128 - } - } -}%% -packet-beta - 0-3: "Magic Number (0xFD, 0x2F, 0xC5, 0x30)" - 4-5: "Patch Version" - 6: "Minor Version" - 7: "Major Version" - 8-15: "Original Uncompressed Size" - 16-23: "Compressed Archive Size" - 24-55: "Reserved Padding" - 56-59: "Metadata Section Size" - 60-61: "Compression Type" - 62-63: "Reserved Padding" -::: +``` +Header { + magic_number: uint8_t[4] = {0xFD, 0x2F, 0xC5, 0x30} + patch_version: uint16_t + minor_version: uint8_t + major_version: uint8_t + original_uncompressed_size: uint64_t + compressed_archive_size: uint64_t + reserved_padding1: uint64_t[4] + metadata_section_size: uint32_t + compression_type: uint16_t + reserved_padding2: uint16_t +} +``` +++ **Figure 2**: Layout of the 64-byte archive header. :::: -The value in `compressed archive size` indicates the total size of the archive, including the size -of the header. The `original uncompressed size` field indicates the total size of the data that was -compressed into the archive. The interpretation of `original uncompressed size` changes somewhat +The value in `compressed_archive_size` indicates the total size of the archive, including the size +of the header. The `original_uncompressed_size` field indicates the total size of the data that was +compressed into the archive. The interpretation of `original_uncompressed_size` changes somewhat based on what data was ingested into the archive. For example, when ingesting `ndjson`, the field records the number of bytes of raw JSON data ingested into the archive --- however, when ingesting KV-IR streams, the field records the number of decompressed bytes of KV-IR ingested into the archive (since KV-IR streams are typically compressed by a general purposed compressor before being stored). -The `metadata section size` field indicates the _compressed_ size of the metadata section in bytes. -This field can be used in combination with the `original uncompressed size` field and known header +The `metadata_section_size` field indicates the _compressed_ size of the metadata section in bytes. +This field can be used in combination with the `original_uncompressed_size` field and known header size to determine the size and offset of both the `metadata` and `files` sections. The compression type for an archive indicates the general-purpose compressor used to compress each section of the archive and is currently one of: * `0x0000` - ZStandard -All `reserved padding` fields are reserved for use in future versions of the single-file archive +All `reserved_padding` fields are reserved for use in future versions of the single-file archive format. ## Metadata section -The `metadata`` section is made up of a compressed sequence of metadata packets encoded with the +The `metadata` section is made up of a compressed sequence of metadata packets encoded with the binary format shown in [Figure 3](#figure-3). Metadata packets generally contain information that relates to the entire archive or large sections of the archive. From d5bebd6d43b803621c4d99869184734dd59e5688 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Tue, 17 Jun 2025 17:42:02 +0000 Subject: [PATCH 20/21] Address more comments --- .../dev-guide/design-clp-structured/index.md | 2 +- .../single-file-archive-format.md | 22 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/src/dev-guide/design-clp-structured/index.md b/docs/src/dev-guide/design-clp-structured/index.md index ee3806a216..205a83c289 100644 --- a/docs/src/dev-guide/design-clp-structured/index.md +++ b/docs/src/dev-guide/design-clp-structured/index.md @@ -15,7 +15,7 @@ resource-constrained usage, but ideal for long-term archival and search. :link: background Background ^^^ -All necessary information needed to understand `clp-s`' format. +All information needed to understand `clp-s`' format. ::: :::{grid-item-card} diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md index 8c6384c648..4b82aab00b 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md @@ -6,7 +6,7 @@ order to enable high performance for archives stored on object storage systems s [S3][amazon-s3]. This documentation records the details of the single-file archive (v0.3.1) format and what it -enables, with minimal discussion of design rationale. For more information about the design +enables, with minimal discussion of design rationale. For more information about the design decisions behind `clp-s`, please refer to [our paper on `clp-s`][μSlope], or our [blog][s3-blog] on optimizing `clp-s` for object storage. @@ -68,7 +68,7 @@ The archive header is a 64-byte unit at the start of a single-file archive that most important metadata information about an archive, as shown in [Figure 2](#figure-2). The header begins with a 4-byte magic number which identifies the file as an archive. The magic number is followed by a 4-byte version number, comprised of a 2-byte patch version number, followed by 1-byte -minor and major version numbers respectively. For version 0.X.Y archives, every minor version +minor and major version numbers respectively. For version 0.X.Y archives, every minor version change is breaking. To reduce maintenance while the archive format stabilizes, not all readers designed for a given minor version are backwards-compatible. @@ -105,7 +105,7 @@ This field can be used in combination with the `original_uncompressed_size` fiel size to determine the size and offset of both the `metadata` and `files` sections. The compression type for an archive indicates the general-purpose compressor used to compress each -section of the archive and is currently one of: +section of the archive, and is currently one of: * `0x0000` - ZStandard All `reserved_padding` fields are reserved for use in future versions of the single-file archive @@ -138,9 +138,9 @@ equivalent to a single instance of `MetadataPacketStream`. Each metadata packet has a type, size, and arbitrary binary payload. Different packet types make different choices for encoding content in this binary payload. Since the size of each packet is -well-defined reader implementations can attempt to read archives containing packet types they are -unfamiliar with by simply skipping the corresponding content. Generally this metadata section design -is intended to allow for some degree of forwards-compatibility and extensibility. +well-defined, reader implementations can attempt to read archives containing packet types they are +unfamiliar with by simply skipping the corresponding content. Generally, this metadata section +design is intended to allow for some degree of forwards-compatibility and extensibility. Archives currently support the following metadata packet types, some of which are optional: * `0x00` - ArchiveInfo @@ -152,8 +152,8 @@ Archives currently support the following metadata packet types, some of which ar The ArchiveInfo packet is a msgpack map that currently only records the number of segments in an archive, as shown in [Figure 4](#figure-4). Each archive currently consists of only a single segment -(i.e. there is only one tables segment file), but we still record the number of segments in order -to offer backwards compatibility if we do start splitting tables into multiple segments. +(i.e. there is only one tables segment file), but we still record the number of segments to offer +backwards compatibility if we do start splitting tables into multiple segments. This metadata packet was originally intended to mimic the "ArchiveMetadata" structure in CLP, but most of what "ArchiveMetadata" records is now present in either the header or the RangeIndex @@ -277,7 +277,7 @@ Note that ingestion units can be split across multiple archives in order to main archive size, and that in these cases the properties become associated with each chunk of the ingestion unit in each archive. -By default we automatically store the following properties about each ingestion unit: +By default, we automatically store the following properties about each ingestion unit: * `"_filename"` - the original filename of the ingestion unit as it was passed to `clp-s` during compression * `"_file_split_number"` - incremented each time this ingestion-unit is split across another archive @@ -310,11 +310,11 @@ The RangeIndex packet is encoded as a msgpack array with the format shown in [Fi **Figure 7**: Layout of the RangeIndex msgpack payload. Each entry in the array records a start index `"s"` and end index `"e"` indicating that the properties correspond to the logical range of records `[s, e)`. No entries in the range index have overlapping logical ranges, and all entries are -ordered by logical range. The `"f"` object contains the properties associated with an ingestion +ordered by logical range. The `"f"` object contains the properties associated with an ingestion unit. :::: -Note that properties created and used by `clp-s` are always prefixed with the `_` character. To +Note that properties created and used by `clp-s` are always prefixed with the `_` character. To avoid naming collisions with properties created by `clp-s`, users should avoid creating properties with this prefix. From 088d483e436dd6724932a7dd35ba5bdec4c51958 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 12 Nov 2025 16:43:25 -0500 Subject: [PATCH 21/21] Fixes after merging main. --- .../{dev-guide => dev-docs}/design-clp-structured/background.md | 0 docs/src/{dev-guide => dev-docs}/design-clp-structured/index.md | 0 .../design-clp-structured/single-file-archive-format.md | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename docs/src/{dev-guide => dev-docs}/design-clp-structured/background.md (100%) rename docs/src/{dev-guide => dev-docs}/design-clp-structured/index.md (100%) rename docs/src/{dev-guide => dev-docs}/design-clp-structured/single-file-archive-format.md (99%) diff --git a/docs/src/dev-guide/design-clp-structured/background.md b/docs/src/dev-docs/design-clp-structured/background.md similarity index 100% rename from docs/src/dev-guide/design-clp-structured/background.md rename to docs/src/dev-docs/design-clp-structured/background.md diff --git a/docs/src/dev-guide/design-clp-structured/index.md b/docs/src/dev-docs/design-clp-structured/index.md similarity index 100% rename from docs/src/dev-guide/design-clp-structured/index.md rename to docs/src/dev-docs/design-clp-structured/index.md diff --git a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md b/docs/src/dev-docs/design-clp-structured/single-file-archive-format.md similarity index 99% rename from docs/src/dev-guide/design-clp-structured/single-file-archive-format.md rename to docs/src/dev-docs/design-clp-structured/single-file-archive-format.md index 4b82aab00b..43a534718e 100644 --- a/docs/src/dev-guide/design-clp-structured/single-file-archive-format.md +++ b/docs/src/dev-docs/design-clp-structured/single-file-archive-format.md @@ -257,7 +257,7 @@ instance of "TimestampDictionary". :::: The key name in each TimestampRange follows the same -[escaping rules](../../user-guide/reference-json-search-syntax.md) we use for key names in KQL +[escaping rules](../../user-docs/reference-json-search-syntax.md) we use for key names in KQL search. Note that we allow each key to map to multiple MPT nodes, and that each key's range can be recorded as either integer epoch time or double epoch time to handle timestamp columns with polymorphic types.