Skip to content

CF-4 hierarchy and cognitive paging #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hardcoreerik merged 2 commits into from
Jun 29, 2026
Merged

CF-4 hierarchy and cognitive paging #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a8821f4
Build CF-4 hierarchy and cognitive paging
hardcoreerik Jun 29, 2026
af555fc
Address CF-4 CodeRabbit findings
hardcoreerik Jun 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions OrchestratorIDE.Avalonia/OrchestratorIDE.Avalonia.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,10 @@
<Compile Include="..\OrchestratorIDE\Services\ContextFabric\DocumentGraphRepository.cs" />
<Compile Include="..\OrchestratorIDE\Services\ContextFabric\FabricEvidenceGraphImporter.cs" />
<Compile Include="..\OrchestratorIDE\Services\ContextFabric\FabricNativeReaderService.cs" />
<Compile Include="..\OrchestratorIDE\Services\ContextFabric\FabricReducer.cs" />
<Compile Include="..\OrchestratorIDE\Services\ContextFabric\FabricQueryPlanner.cs" />
<Compile Include="..\OrchestratorIDE\Services\ContextFabric\EvidencePackBuilder.cs" />
<Compile Include="..\OrchestratorIDE\Services\ContextFabric\FabricCitationVerifier.cs" />
<Compile Include="..\OrchestratorIDE\Services\ContextFabric\FabricSearchService.cs" />
<!-- CodeGraph v1 (step 1+) — shared with WPF via explicit include (Avalonia_Migration.md discipline) -->
<Compile Include="..\OrchestratorIDE\Services\CodeGraph\GraphModels.cs" />
Expand Down
560 changes: 560 additions & 0 deletions OrchestratorIDE.UnitTests/ContextFabricCf4Tests.cs
View file
Open in desktop

Large diffs are not rendered by default.

131 changes: 131 additions & 0 deletions OrchestratorIDE/Services/ContextFabric/ContextFabricIngestionContracts.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,28 @@ public static class FabricVerificationStatus
public const string Rejected = "rejected";
}

public static class FabricCoverageStatus
{
public const string Complete = "complete";
public const string Incomplete = "incomplete";
}

public static class FabricQueryMode
{
public const string Quick = "quick";
public const string Study = "study";
}

public static class FabricCitationVerificationLabel
{
public const string Supported = "supported";
public const string PartiallySupported = "partially_supported";
public const string Contradicted = "contradicted";
public const string CitationMismatch = "citation_mismatch";
public const string Interpretive = "interpretive";
public const string Unverifiable = "unverifiable";
}

public sealed record FabricParsedBlock(
int CharStart,
int CharEnd,
Expand Down Expand Up @@ -164,6 +186,115 @@ public sealed record FabricRetrievalHit(
string? ClaimText,
string? VerificationStatus);

public sealed record FabricMemoryNodeEntry(
string NodeId,
string CorpusId,
string DocumentId,
string NodeType,
string Title,
string SummaryText,
int Generation,
int FanIn,
int ExpectedChildCount,
int CoveredChildCount,
string CoverageStatus,
string ReducerVersion,
DateTimeOffset CreatedAt,
DateTimeOffset UpdatedAt);

public sealed record FabricMemoryMembershipEntry(
string ParentNodeId,
string ChildKind,
string ChildId,
int Ordinal,
bool IsCovered);

public sealed record FabricReducerOptions(
int FanIn = 4,
int MaxSummaryChars = 320)
{
public void Validate()
{
if (FanIn < 2 || FanIn > 16)
throw new ArgumentOutOfRangeException(nameof(FanIn));
if (MaxSummaryChars < 64)
throw new ArgumentOutOfRangeException(nameof(MaxSummaryChars));
}
}

public sealed record FabricReductionResult(
string DocumentId,
IReadOnlyList<FabricMemoryNodeEntry> Nodes,
IReadOnlyList<FabricMemoryMembershipEntry> Memberships,
string RootNodeId);

public sealed record FabricQueryPlannerOptions(
int RetrievalLimit = 8,
int MaxRounds = 2,
int MaxSourceOpens = 6,
int MaxPromptTokens = 8_192,
int ResponseTokenReserve = 1_024)
{
public void Validate()
{
if (RetrievalLimit < 1 || RetrievalLimit > 64)
throw new ArgumentOutOfRangeException(nameof(RetrievalLimit));
if (MaxRounds < 1 || MaxRounds > 8)
throw new ArgumentOutOfRangeException(nameof(MaxRounds));
if (MaxSourceOpens < 1 || MaxSourceOpens > 64)
throw new ArgumentOutOfRangeException(nameof(MaxSourceOpens));
if (ResponseTokenReserve < 128 || ResponseTokenReserve >= MaxPromptTokens)
throw new ArgumentOutOfRangeException(nameof(ResponseTokenReserve));
}
}

public sealed record FabricQueryPlan(
string Query,
string CorpusId,
string Mode,
int MaxRounds,
int MaxSourceOpens,
int MaxPromptTokens,
int ResponseTokenReserve,
IReadOnlyList<FabricRetrievalHit> SeedHits,
IReadOnlyList<string> SummaryNodeIds,
IReadOnlyList<string> ReopenedSegmentIds,
bool TriggeredSourceReopen);

public sealed record FabricEvidenceItem(
string Kind,
string Id,
string Text,
int TokenCount,
bool FromSource,
string Provenance);

public sealed record FabricEvidencePack(
string Query,
string Mode,
int PromptTokenBudget,
int ResponseTokenReserve,
int UsedPromptTokens,
IReadOnlyList<FabricEvidenceItem> Included,
IReadOnlyList<FabricEvidenceItem> Excluded,
bool WithinBudget,
bool TriggeredSourceReopen);

public sealed record FabricCitationVerificationItem(
string SegmentId,
string Label,
string QuoteText,
int CharStart,
int CharEnd,
string Reason);

public sealed record FabricCitationVerificationResult(
string ClaimText,
string Label,
IReadOnlyList<FabricCitationVerificationItem> Items,
bool Repaired,
IReadOnlyList<FabricCitation> EffectiveCitations);

public sealed record FabricSegmenterOptions(
int TargetTokens = 2_000,
int MaximumTokens = 3_000,
Expand Down
202 changes: 202 additions & 0 deletions OrchestratorIDE/Services/ContextFabric/DocumentGraphRepository.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,23 @@ FROM fabric_claims
P(ps, "$limit", Math.Clamp(limit, 1, 500));
});

public IReadOnlyList<FabricClaimEntry> ListClaimsForDocument(string documentId, string? verificationStatus = null, int limit = 500) => Query(
"""
SELECT *
FROM fabric_claims
WHERE document_id = $document
AND ($status IS NULL OR verification_status = $status)
ORDER BY segment_id, claim_id
LIMIT $limit
""",
MapClaim,
ps =>
{
P(ps, "$document", documentId);
P(ps, "$status", verificationStatus);
P(ps, "$limit", Math.Clamp(limit, 1, 4096));
});

public IReadOnlyList<FabricClaimCitationEntry> ListClaimCitations(string claimId) => Query(
"""
SELECT *
Expand Down Expand Up @@ -243,6 +260,82 @@ FROM fabric_relations
P(ps, "$limit", Math.Clamp(limit, 1, 500));
});

public void ReplaceMemoryNodesForDocument(
string documentId,
IReadOnlyList<FabricMemoryNodeEntry> nodes,
IReadOnlyDictionary<string, IReadOnlyList<FabricMemoryMembershipEntry>> membershipsByParentId)
{
if (string.IsNullOrWhiteSpace(documentId))
throw new ArgumentException("Document id is required.", nameof(documentId));
ArgumentNullException.ThrowIfNull(nodes);
ArgumentNullException.ThrowIfNull(membershipsByParentId);
if (nodes.Any(node => !string.Equals(node.DocumentId, documentId, StringComparison.Ordinal)))
throw new InvalidDataException($"Replacement memory nodes must all belong to document '{documentId}'.");
ValidateMemoryMemberships(documentId, nodes, membershipsByParentId);

InTransaction((conn, tx) =>
{
ExecuteOn(tx, """
DELETE FROM fabric_memory_nodes
WHERE document_id = $document
""",
ps => P(ps, "$document", documentId));

foreach (var node in nodes.OrderBy(item => item.Generation).ThenBy(item => item.NodeId))
{
InsertMemoryNodeOn(conn, tx, node);
if (!membershipsByParentId.TryGetValue(node.NodeId, out var memberships))
continue;

foreach (var membership in memberships.OrderBy(item => item.Ordinal))
InsertMemoryMembershipOn(conn, tx, membership);
}
});
}

public FabricMemoryNodeEntry? GetMemoryNode(string nodeId) => Query(
"SELECT * FROM fabric_memory_nodes WHERE node_id = $id",
MapMemoryNode,
ps => P(ps, "$id", nodeId)).SingleOrDefault();

public IReadOnlyList<FabricMemoryNodeEntry> ListMemoryNodes(
string corpusId,
string? documentId = null,
int? generation = null,
int limit = 200) => Query(
"""
SELECT *
FROM fabric_memory_nodes
WHERE corpus_id = $corpus
AND ($document IS NULL OR document_id = $document)
AND ($generation IS NULL OR generation = $generation)
ORDER BY generation DESC, node_id
LIMIT $limit
""",
MapMemoryNode,
ps =>
{
P(ps, "$corpus", corpusId);
P(ps, "$document", documentId);
P(ps, "$generation", generation);
P(ps, "$limit", Math.Clamp(limit, 1, 1000));
});

public IReadOnlyList<FabricMemoryMembershipEntry> ListMemoryMemberships(string parentNodeId) => Query(
"""
SELECT *
FROM fabric_memory_memberships
WHERE parent_node_id = $parent
ORDER BY ordinal
""",
reader => new FabricMemoryMembershipEntry(
reader.GetString(reader.GetOrdinal("parent_node_id")),
reader.GetString(reader.GetOrdinal("child_kind")),
reader.GetString(reader.GetOrdinal("child_id")),
reader.GetInt32(reader.GetOrdinal("ordinal")),
reader.GetInt32(reader.GetOrdinal("is_covered")) == 1),
ps => P(ps, "$parent", parentNodeId));

private static FabricClaimEntry MapClaim(SqliteDataReader reader) => new(
reader.GetString(reader.GetOrdinal("claim_id")),
reader.GetString(reader.GetOrdinal("corpus_id")),
Expand All @@ -255,10 +348,73 @@ FROM fabric_relations
DateTimeOffset.Parse(reader.GetString(reader.GetOrdinal("created_at"))),
DateTimeOffset.Parse(reader.GetString(reader.GetOrdinal("updated_at"))));

private static FabricMemoryNodeEntry MapMemoryNode(SqliteDataReader reader) => new(
reader.GetString(reader.GetOrdinal("node_id")),
reader.GetString(reader.GetOrdinal("corpus_id")),
reader.GetString(reader.GetOrdinal("document_id")),
reader.GetString(reader.GetOrdinal("node_type")),
reader.GetString(reader.GetOrdinal("title")),
reader.GetString(reader.GetOrdinal("summary_text")),
reader.GetInt32(reader.GetOrdinal("generation")),
reader.GetInt32(reader.GetOrdinal("fan_in")),
reader.GetInt32(reader.GetOrdinal("expected_child_count")),
reader.GetInt32(reader.GetOrdinal("covered_child_count")),
reader.GetString(reader.GetOrdinal("coverage_status")),
reader.GetString(reader.GetOrdinal("reducer_version")),
DateTimeOffset.Parse(reader.GetString(reader.GetOrdinal("created_at"))),
DateTimeOffset.Parse(reader.GetString(reader.GetOrdinal("updated_at"))));

private static string BuildFtsQuery(string query) => string.Join(" AND ", query
.Split([' ', '\t', '\r', '\n'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.Select(term => $"\"{term.Replace("\"", "\"\"")}\""));

private void ValidateMemoryMemberships(
string documentId,
IReadOnlyList<FabricMemoryNodeEntry> nodes,
IReadOnlyDictionary<string, IReadOnlyList<FabricMemoryMembershipEntry>> membershipsByParentId)
{
var replacementNodes = nodes.ToDictionary(node => node.NodeId, StringComparer.Ordinal);

foreach (var parentId in membershipsByParentId.Keys)
{
if (!replacementNodes.ContainsKey(parentId))
throw new InvalidDataException($"Memory memberships reference missing parent node '{parentId}'.");
}

foreach (var (parentId, memberships) in membershipsByParentId)
{
var seenOrdinals = new HashSet<int>();
foreach (var membership in memberships)
{
if (!string.Equals(membership.ParentNodeId, parentId, StringComparison.Ordinal))
throw new InvalidDataException($"Memory membership parent '{membership.ParentNodeId}' does not match '{parentId}'.");
if (membership.ChildKind is not ("segment" or "memory"))
throw new InvalidDataException($"Memory membership child kind '{membership.ChildKind}' is not supported.");
if (!seenOrdinals.Add(membership.Ordinal))
throw new InvalidDataException($"Memory memberships for '{parentId}' contain duplicate ordinal '{membership.Ordinal}'.");

if (membership.ChildKind == "segment")
{
var segment = Query(
"SELECT document_id FROM fabric_segments WHERE segment_id = $id",
reader => reader.GetString(reader.GetOrdinal("document_id")),
ps => P(ps, "$id", membership.ChildId)).SingleOrDefault();
if (segment is null)
throw new InvalidDataException($"Memory membership references missing segment '{membership.ChildId}'.");
if (!string.Equals(segment, documentId, StringComparison.Ordinal))
throw new InvalidDataException($"Memory membership segment '{membership.ChildId}' does not belong to document '{documentId}'.");
}
else
{
if (!replacementNodes.TryGetValue(membership.ChildId, out var childNode))
throw new InvalidDataException($"Memory membership references missing child node '{membership.ChildId}'.");
if (!string.Equals(childNode.DocumentId, documentId, StringComparison.Ordinal))
throw new InvalidDataException($"Memory membership child node '{membership.ChildId}' does not belong to document '{documentId}'.");
}
}
}
}

private static void UpsertClaimOn(SqliteConnection conn, SqliteTransaction tx, FabricClaimEntry claim)
{
using var cmd = CreateCmd(conn, tx, """
Expand Down Expand Up @@ -312,4 +468,50 @@ INSERT INTO fabric_claim_citations
P(cmd.Parameters, "$quote", citation.QuoteText);
cmd.ExecuteNonQuery();
}

private static void InsertMemoryNodeOn(SqliteConnection conn, SqliteTransaction tx, FabricMemoryNodeEntry node)
{
using var cmd = CreateCmd(conn, tx, """
INSERT INTO fabric_memory_nodes
(node_id, corpus_id, document_id, node_type, title, summary_text, generation, fan_in,
expected_child_count, covered_child_count, coverage_status, reducer_version, created_at, updated_at)
VALUES
($id, $corpus, $document, $type, $title, $summary, $generation, $fanIn,
$expected, $covered, $status, $version, $created, $updated)
""");
P(cmd.Parameters, "$id", node.NodeId);
P(cmd.Parameters, "$corpus", node.CorpusId);
P(cmd.Parameters, "$document", node.DocumentId);
P(cmd.Parameters, "$type", node.NodeType);
P(cmd.Parameters, "$title", node.Title);
P(cmd.Parameters, "$summary", node.SummaryText);
P(cmd.Parameters, "$generation", node.Generation);
P(cmd.Parameters, "$fanIn", node.FanIn);
P(cmd.Parameters, "$expected", node.ExpectedChildCount);
P(cmd.Parameters, "$covered", node.CoveredChildCount);
P(cmd.Parameters, "$status", node.CoverageStatus);
P(cmd.Parameters, "$version", node.ReducerVersion);
P(cmd.Parameters, "$created", node.CreatedAt.ToString("O"));
P(cmd.Parameters, "$updated", node.UpdatedAt.ToString("O"));
cmd.ExecuteNonQuery();
}

private static void InsertMemoryMembershipOn(
SqliteConnection conn,
SqliteTransaction tx,
FabricMemoryMembershipEntry membership)
{
using var cmd = CreateCmd(conn, tx, """
INSERT INTO fabric_memory_memberships
(parent_node_id, child_kind, child_id, ordinal, is_covered)
VALUES
($parent, $kind, $child, $ordinal, $covered)
""");
P(cmd.Parameters, "$parent", membership.ParentNodeId);
P(cmd.Parameters, "$kind", membership.ChildKind);
P(cmd.Parameters, "$child", membership.ChildId);
P(cmd.Parameters, "$ordinal", membership.Ordinal);
P(cmd.Parameters, "$covered", membership.IsCovered ? 1 : 0);
cmd.ExecuteNonQuery();
}
}
Loading
Loading