Skip to content

Commit 87685b6

Browse files
danlannan777danlannan777
committed
checkpoint: matches running, slight memory leak
1 parent 2608024 commit 87685b6

19 files changed

+1908
-109
lines changed

‎CHANGELOG.md

+7
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
## [1.2.5](https://github.com/ThinAirThings/uix/compare/v1.2.4...v1.2.5) (2024-05-23)
22

3+
## 2.4.0
4+
5+
### Minor Changes
6+
7+
- Added correct path resolution for config
8+
Fixed type issue with setting MatchToRelationshipType
9+
310
## 2.3.4
411

512
### Patch Changes

‎package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"name": "@thinairthings/uix",
33
"author": "Dan Lannan",
4-
"version": "2.3.4",
4+
"version": "2.4.0",
55
"type": "module",
66
"types": "./dist/lib/index.d.ts",
77
"bin": {
@@ -15,6 +15,7 @@
1515
"build": "tsup-node",
1616
"build:run": "tsup-node && node dist/cli/cli.js",
1717
"test": "dotenvx run -f .env.test -- pnpm build && dotenvx run -f .env.test -- vitest run basic --test-timeout=100000",
18+
"test:match": "pnpm build && dotenvx run -f .env.test -- vitest run match --test-timeout=100000",
1819
"uix": "dotenvx run -f .env.test -- node ./dist/cli/cli.js --config=./tests/uix/uix.config.ts",
1920
"wipset": "changeset && changeset version && changeset publish"
2021
},

‎src/fns/deleteNodeFactory.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ export const deleteNodeFactory = <
4646
...nodeKey
4747
});
4848
}
49-
console.log("Deleted", parentNodeKeys)
49+
console.log("Deleted", nodeKey)
5050
if (!parentNodeKeys.length) return UixErr({
5151
subtype: UixErrSubtype.DELETE_NODE_FAILED,
5252
message: `Failed to delete node of type ${nodeKey.nodeType as string} with id ${nodeKey.nodeId}`,

‎src/templates/staticObjectsTemplate.ts

+5-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ export const staticObjectsTemplate = (config: GenericUixConfig) => {
66
return /* ts */`
77
// Start of File
88
import uixConfig from '${config.pathToConfig.replace('uix.config.ts', 'uix.config')}'
9-
import { NodeShape } from '@thinairthings/uix'
9+
import { NodeShape, NodeState } from '@thinairthings/uix'
1010
import neo4j from 'neo4j-driver'
1111
export type ConfiguredNodeTypeMap = typeof uixConfig.graph.nodeTypeMap
1212
@@ -22,6 +22,10 @@ ${config.graph.nodeTypeMap['Root']
2222
${Object.keys(config.graph.nodeTypeMap).map(nodeType =>
2323
/*ts*/`export type ${nodeType}Node = NodeShape<ConfiguredNodeTypeMap['${nodeType}']> \n`
2424
).join('')}
25+
${Object.keys(config.graph.nodeTypeMap).map(nodeType =>
26+
/*ts*/`export type ${nodeType}NodeState = NodeState<ConfiguredNodeTypeMap['${nodeType}']> \n`
27+
).join('')
28+
}
2529
2630
export const driver = neo4j.driver(
2731
process.env.NEO4J_URI!,

‎src/vectors/upsertMatchToRelationship.ts ‎src/vectors/upsertMatchTypeEmbedding.ts

+44-19
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,47 @@
11
import OpenAI from "openai"
2-
import { AnyNodeShape, GenericMatchToRelationshipType, GenericNodeShape, GenericNodeType } from "../types/NodeType"
2+
import { GenericNodeShape, GenericNodeType } from "../types/NodeType"
33
import { Driver, EagerResult, Integer, Node } from "neo4j-driver"
44
import { Ok, UixErr, UixErrSubtype } from "../types/Result"
55
import { openAIAction } from "../clients/openai"
66
import { neo4jAction } from "../clients/neo4j"
7+
import dedent from "dedent"
8+
import { GenericMatchToRelationshipType } from "../types/MatchToRelationshipType"
79

810

911

10-
export const upsertMatchToRelationship = async (
12+
export const upsertMatchTypeEmbedding = async (
1113
neo4jDriver: Driver,
1214
openaiClient: OpenAI,
13-
nodeShape: AnyNodeShape,
14-
matchToRelationshipType: GenericMatchToRelationshipType,
15-
matchToNodeType: GenericNodeType
15+
triggerNode: GenericNodeShape,
16+
fromNodeType: GenericNodeType,
17+
matchToRelationshipType: GenericMatchToRelationshipType
1618
) => {
1719
// Try/catch this because you're not going to handle it with application logic.
1820
// You'll just log it.
1921
const result = await neo4jAction(openAIAction(async () => {
2022
// Create Node Type Summary
21-
console.log("Creating Node Type Summary", nodeShape)
23+
console.log("Creating Match Type Summary (nodeTypeEmbedding)", fromNodeType.type)
24+
// Collect all nodes in weightedNodeTypeSet
25+
const {
26+
targetNode,
27+
weightedNodeSet
28+
} = await neo4jDriver.executeQuery<EagerResult<{
29+
weightedNode: Node<Integer, GenericNodeShape>,
30+
targetNode: Node<Integer, GenericNodeShape>
31+
}>>(dedent/*cypher*/`
32+
// Get the node that initiated the change as a reference point
33+
match (triggerNode:${triggerNode.nodeType} {nodeId: $triggerNode.nodeId})
34+
// Get the parent node that is the target of this update
35+
match (targetNode: ${fromNodeType.type})<-[:CHILD_TO|UNIQUE_TO*]-(triggerNode)
36+
// Get all nodes in the weightedNodeTypeSet
37+
match (targetNode)<-[:CHILD_TO|UNIQUE_TO*]-(weightedNode: ${matchToRelationshipType.weightedNodeTypeSet.map(({ NodeType }) => NodeType.type).join('|')})
38+
return weightedNode, targetNode
39+
`, {
40+
triggerNode
41+
}).then(res => ({
42+
targetNode: res.records[0].get('targetNode').properties,
43+
weightedNodeSet: res.records.map(record => record.get('weightedNode').properties)
44+
}))
2245
const { type, description } = matchToRelationshipType
2346
const nodeTypeSummary = await openaiClient.chat.completions.create({
2447
model: 'gpt-4o',
@@ -34,43 +57,45 @@ export const upsertMatchToRelationship = async (
3457
+ `You should ignore information that is not relevant to the '${type.toUpperCase()}Type' paragraph as it was defined.\n`
3558
}, {
3659
role: 'user',
37-
content: `The JSON data to use is: ${JSON.stringify(nodeShape)}`
60+
content: `The JSON data to use is: ${JSON.stringify(Object.fromEntries(weightedNodeSet.map(node => ([node.nodeType, node]))))}`
3861
}
3962
]
4063
}).then(res => res.choices[0].message.content ?? '')
64+
console.log("Target Node", targetNode)
65+
console.log("Weighted Node Set", weightedNodeSet)
66+
console.log("Node Type Summary", nodeTypeSummary)
4167
const nodeTypeEmbedding = await openaiClient.embeddings.create({
4268
model: 'text-embedding-3-large',
4369
input: nodeTypeSummary
4470
}).then(res => res.data[0].embedding)
4571
// Update Node
46-
const nodeResult = await neo4jDriver.executeQuery<EagerResult<{
47-
toNode: Node<Integer, GenericNodeShape>
72+
const vectorNode = await neo4jDriver.executeQuery<EagerResult<{
73+
vectorNode: Node<Integer, GenericNodeShape>
4874
}>>(/*cypher*/`
49-
match (node:${nodeShape.nodeType} {nodeId: $nodeId})
50-
merge (vectorNode:${nodeShape.nodeType}Vector:${matchToRelationshipType.type} {nodeId: $nodeId})-[:VECTOR_TO]->(node)
75+
match (targetNode:${targetNode.nodeType} {nodeId: $targetNode.nodeId})
76+
merge (vectorNode:${targetNode.nodeType}Vector:${matchToRelationshipType.type} {nodeId: $targetNode.nodeId})-[:VECTOR_TO]->(targetNode)
5177
on create
5278
set vectorNode += $vectorNodeStructure
5379
on match
5480
set vectorNode += $vectorNodeStructure
55-
with node
56-
match(toNode: ${matchToNodeType.type})<-[:CHILD_TO|UNIQUE_TO*]-(node)
57-
return toNode
81+
return vectorNode
5882
`, {
59-
nodeId: nodeShape.nodeId,
83+
targetNode,
6084
vectorNodeStructure: {
6185
nodeTypeSummary,
6286
nodeTypeEmbedding
6387
}
64-
}).then(res => res.records[0].get('toNode').properties)
65-
if (!nodeResult) return UixErr({
88+
}).then(res => res.records[0].get('vectorNode').properties)
89+
console.log("Upserted Match Type Embedding", vectorNode)
90+
if (!vectorNode) return UixErr({
6691
subtype: UixErrSubtype.UPDATE_NODE_FAILED,
6792
message: `Upsert match relationship error`,
6893
data: {
69-
nodeShape,
94+
targetNode,
7095
matchToRelationshipType
7196
}
7297
});
73-
return Ok(nodeResult)
98+
return Ok(targetNode)
7499
}))()
75100
const { data, error } = result
76101
if (data) return data

‎src/vectors/upsertMatches.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { Driver } from "neo4j-driver";
2-
import { GenericMatchToRelationshipType, GenericNodeShape } from "../types/NodeType";
2+
import { GenericNodeShape } from "../types/NodeType";
33
import dedent from 'dedent'
4+
import { GenericMatchToRelationshipType } from "../types/MatchToRelationshipType";
45

56

67

‎src/vectors/upsertMatchesv2.ts

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
import { Driver, EagerResult, Integer, Node } from "neo4j-driver";
2+
import { GenericNodeShape } from "../types/NodeType";
3+
import { GenericMatchToRelationshipType } from "../types/MatchToRelationshipType";
4+
import dedent from "dedent";
5+
6+
7+
8+
9+
export const upsertMatchesv2 = async (
10+
neo4jDriver: Driver,
11+
fromNode: GenericNodeShape,
12+
matchToRelationshipType: GenericMatchToRelationshipType
13+
) => {
14+
// Run Query
15+
const matches = await neo4jDriver.executeQuery<EagerResult<{
16+
score: number,
17+
matchToNode: Node<Integer, GenericNodeShape>
18+
}>>(dedent/*cypher*/`
19+
match (fromNode: ${fromNode.nodeType} {nodeId: $fromNode.nodeId})<-[:VECTOR_TO]-(fromNodeVectorNode:${fromNode.nodeType}Vector:${matchToRelationshipType.type})
20+
call db.index.vector.queryNodes('${matchToRelationshipType.matchToNodeType.type}_vector', 5, fromNodeVectorNode.nodeTypeEmbedding)
21+
yield node as matchToVectorNode, score
22+
match (matchToNode: ${matchToRelationshipType.matchToNodeType.type} {nodeId: matchToVectorNode.nodeId})
23+
merge (fromNode)-[match_to:MATCH_TO { type: $matchToType }]->(matchToNode)
24+
on create
25+
set
26+
match_to.fromNodeId = $fromNode.nodeId,
27+
match_to.type = $matchToType,
28+
match_to.toNodeId = matchToNode.nodeId,
29+
match_to.createdAt = timestamp(),
30+
match_to.updatedAt = timestamp(),
31+
match_to.score = score
32+
on match
33+
set match_to.updatedAt = timestamp(),
34+
match_to.score = score
35+
return score, matchToNode
36+
`, {
37+
fromNode,
38+
matchToType: matchToRelationshipType.matchToNodeType.type
39+
}).then(res => res.records.map(record => ({
40+
score: record.get('score'),
41+
matchToNode: record.get('matchToNode').properties
42+
})))
43+
44+
console.log(matches)
45+
return matches
46+
}

‎src/vectors/upsertVectorNode.ts

+46-32
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
import { Driver } from "neo4j-driver";
22
import OpenAI from "openai";
33
import { AnyNodeShape, GenericNodeType, GenericNodeTypeMap } from "../types/NodeType";
4-
import { upsertMatchToRelationship } from "./upsertMatchToRelationship";
54
import { upsertPropertyVector } from "./upsertPropertyVector";
65
import { upsertMatches } from "./upsertMatches";
6+
import { match } from "assert";
7+
import { upsertMatchTypeEmbedding } from "./upsertMatchTypeEmbedding";
8+
import { upsertMatchesv2 } from "./upsertMatchesv2";
79

810
export const upsertVectorNode = async (
911
neo4jDriver: Driver,
@@ -21,36 +23,48 @@ export const upsertVectorNode = async (
2123
nodeShape
2224
))
2325
])
24-
// This gathers the matchToRelationshipTypes from the graph structure that are relevant to this specific node
25-
const matchToRelationshipComponents = Object.values(nodeTypeMap).map(nodeType =>
26-
({
27-
nodeType,
28-
matchNodeTypes: nodeType.matchToRelationshipTypeSet.filter(matchToRelationshipType =>
29-
[matchToRelationshipType.matchToNodeType.type, ...matchToRelationshipType.weightedNodeTypeSet.map(({ NodeType }) => NodeType.type)]
30-
.some(matchNodeType => nodeShape.nodeType === matchNodeType)
31-
).flat()
32-
})
33-
)
34-
// Handle Vector Operations
35-
const toNodes = await Promise.all([
36-
// Create Summary in background
37-
...matchToRelationshipComponents.length
38-
? matchToRelationshipComponents
39-
.map(({ nodeType, matchNodeTypes }) => matchNodeTypes.flatMap(async matchNodeType => await upsertMatchToRelationship(
40-
neo4jDriver,
41-
openaiClient,
42-
nodeShape,
43-
matchNodeType,
44-
nodeType
45-
))).flat()
46-
: [],
47-
])
48-
// // Note: You need to figure out how to get a hold of the fromNode
49-
if (!toNodes.length) return
50-
await Promise.all(toNodes.map(async toNode => await upsertMatches(
51-
neo4jDriver,
52-
toNode!,
53-
nodeTypeMap[toNode!.nodeType].matchToRelationshipTypeSet[0]
54-
)))
26+
// Check all possible fromNodeTypes to see which ones are associated with the nodeShape in question.
27+
const fromNodeTypesAndRelationshipTypesToDirectlyUpdate = Object.values(nodeTypeMap).map(fromNodeType => ({
28+
fromNodeType,
29+
matchToRelationshipTypeSet: fromNodeType.matchToRelationshipTypeSet.filter(matchToRelationshipType =>
30+
matchToRelationshipType.weightedNodeTypeSet.some(({ NodeType }) => NodeType.type === nodeShape.nodeType)
31+
)
32+
}))
33+
console.log(fromNodeTypesAndRelationshipTypesToDirectlyUpdate)
34+
const fromNodeTypesToUpdateMatches = Object.values(nodeTypeMap).map(fromNodeType => ({
35+
fromNodeType,
36+
matchToRelationshipTypeSet: fromNodeType.matchToRelationshipTypeSet.filter(matchToRelationshipType =>
37+
matchToRelationshipType.weightedNodeTypeSet.some(({ NodeType }) => NodeType.type === nodeShape.nodeType)
38+
|| matchToRelationshipType.matchToNodeType.type === nodeShape.nodeType
39+
)
40+
}))
41+
// Update all nodes that have a direct relationship to the nodeShape
42+
const updatedTargetNodes = await Promise.all([
43+
...fromNodeTypesAndRelationshipTypesToDirectlyUpdate.map(async ({ fromNodeType, matchToRelationshipTypeSet }) =>
44+
await Promise.all(matchToRelationshipTypeSet.map(async matchToRelationshipType => await upsertMatchTypeEmbedding(
45+
neo4jDriver,
46+
openaiClient,
47+
nodeShape,
48+
fromNodeType,
49+
matchToRelationshipType
50+
)))
51+
)
52+
]).then(res => res.flat().filter(node => node !== undefined))
53+
console.log("Updated Target Nodes", updatedTargetNodes)
54+
// Add matches to targetNode
55+
await Promise.all(updatedTargetNodes.map(async (targetNode) =>
56+
await Promise.all(nodeTypeMap[targetNode.nodeType].matchToRelationshipTypeSet.map(async matchToRelationshipType => await upsertMatchesv2(
57+
neo4jDriver,
58+
targetNode,
59+
matchToRelationshipType
60+
)))
61+
))
5562

63+
// // // Note: You need to figure out how to get a hold of the fromNode
64+
// if (!toNodes.length) return
65+
// await Promise.all(toNodes.map(async toNode => await upsertMatches(
66+
// neo4jDriver,
67+
// toNode!,
68+
// nodeTypeMap[toNode!.nodeType].matchToRelationshipTypeSet[0]
69+
// )))
5670
}

‎tests/basic.test.ts

+2-13
Original file line numberDiff line numberDiff line change
@@ -45,18 +45,7 @@ test('Integration test', async () => {
4545
degree: 'Master of Arts (M.A.)',
4646
fieldOfStudy: 'Electrical Engineering',
4747
})
48-
const educationNodeSet = Array.from({ length: 5 }).map(async (_, i) => await createNode(
49-
[userNode],
50-
'Education', {
51-
school: 'RPI',
52-
graduationYear: 2022,
53-
description: 'I studied math',
54-
degree: 'Master of Arts (M.A.)',
55-
fieldOfStudy: 'Electrical Engineering',
56-
}))
57-
await Promise.all(educationNodeSet)
5848
expect(createdEducationNode).toBeTruthy()
59-
6049
if (createEducationNodeError) {
6150
console.error(createEducationNodeError)
6251
expect(createEducationNodeError).toBeFalsy()
@@ -94,7 +83,6 @@ test('Integration test', async () => {
9483
expect(updatedEducationVectorNode).toBeTruthy()
9584
expect(createdEducationVectorNode.description![0]).not.toEqual(updatedEducationVectorNode.description![0])
9685

97-
9886
// Check getAllNodeType
9987
const { data: allEducationNodes, error: getAllEducationNodesError } = await getAllOfNodeType('Education', {
10088
limit: 4
@@ -135,7 +123,8 @@ test('Integration test', async () => {
135123
return
136124
}
137125
expect(userNodeByIndex).toBeTruthy()
138-
await new Promise(res => setTimeout(res, 1000 * 20))
126+
writeFileSync(path.resolve('tests', 'userNode.json'), JSON.stringify(userNodeByIndex, null, 2))
127+
await new Promise(res => setTimeout(res, 1000 * 40))
139128
// Check deleteNode
140129
const { data: deleted, error: deleteError } = await deleteNode(userNodeByIndex)
141130
if (deleteError) {

0 commit comments

Comments
 (0)