Skip to content

Commit

Permalink
feat(api-core): support label & property filtering for both edge and …
Browse files Browse the repository at this point in the history 
…vertex & support kout dfs mode (#2295)

- Support label & property filtering for both edge and vertex and the filtering is implemented in Kout Post and Kneighbor - Post Apis, reducing unnecessary graph searches through pruning
- Support Kout dfs mode in Kout Post Api

Originally only edge label filtering was supported, now label and property filtering for edge and vertex is supported.
- add classes VEStepEntity and VEStep to support serialization in request
- add class Steps to support filtering of edge and vertex in runtime(core)
- add new method edgesOfVertex(Id source, Steps steps) to support label and property filtering for both edge and vertex in HugeTraverser.java

---------

Co-authored-by: imbajin <[email protected]>
  • Loading branch information
@DanGuge @imbajin
DanGuge and imbajin authored Sep 10, 2023
1 parent 1d0969d commit 8e99d85
Show file tree
Hide file tree
Showing 15 changed files with 782 additions and 82 deletions.
20 changes: 10 additions & 10 deletions hugegraph-api/src/main/java/org/apache/hugegraph/api/traversers/KneighborAPI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
import org.apache.hugegraph.traversal.algorithm.HugeTraverser;
import org.apache.hugegraph.traversal.algorithm.KneighborTraverser;
import org.apache.hugegraph.traversal.algorithm.records.KneighborRecords;
import org.apache.hugegraph.traversal.algorithm.steps.EdgeStep;
import org.apache.hugegraph.traversal.algorithm.steps.Steps;
import org.apache.hugegraph.type.define.Directions;
import org.apache.hugegraph.util.E;
import org.apache.hugegraph.util.Log;
Expand Down Expand Up @@ -120,17 +120,17 @@ public String post(@Context GraphManager manager,
E.checkArgumentNotNull(request, "The request body can't be null");
E.checkArgumentNotNull(request.source,
"The source of request can't be null");
E.checkArgument(request.step != null,
E.checkArgument(request.steps != null,
"The steps of request can't be null");
if (request.countOnly) {
E.checkArgument(!request.withVertex && !request.withPath && !request.withEdge,
"Can't return vertex, edge or path when count only");
}

LOG.debug("Graph [{}] get customized kneighbor from source vertex " +
"'{}', with step '{}', limit '{}', count_only '{}', " +
"'{}', with steps '{}', limit '{}', count_only '{}', " +
"with_vertex '{}', with_path '{}' and with_edge '{}'",
graph, request.source, request.step, request.limit,
graph, request.source, request.steps, request.limit,
request.countOnly, request.withVertex, request.withPath,
request.withEdge);

Expand All @@ -139,11 +139,11 @@ public String post(@Context GraphManager manager,
HugeGraph g = graph(manager, graph);
Id sourceId = HugeVertex.getIdValue(request.source);

EdgeStep step = step(g, request.step);
Steps steps = steps(g, request.steps);

KneighborRecords results;
try (KneighborTraverser traverser = new KneighborTraverser(g)) {
results = traverser.customizedKneighbor(sourceId, step,
results = traverser.customizedKneighbor(sourceId, steps,
request.maxDepth,
request.limit);
measure.addIterCount(traverser.vertexIterCounter.get(),
Expand Down Expand Up @@ -202,8 +202,8 @@ private static class Request {

@JsonProperty("source")
public Object source;
@JsonProperty("step")
public TraverserAPI.Step step;
@JsonProperty("steps")
public TraverserAPI.VESteps steps;
@JsonProperty("max_depth")
public int maxDepth;
@JsonProperty("limit")
Expand All @@ -219,9 +219,9 @@ private static class Request {

@Override
public String toString() {
return String.format("PathRequest{source=%s,step=%s,maxDepth=%s" +
return String.format("PathRequest{source=%s,steps=%s,maxDepth=%s" +
"limit=%s,countOnly=%s,withVertex=%s," +
"withPath=%s,withEdge=%s}", this.source, this.step,
"withPath=%s,withEdge=%s}", this.source, this.steps,
this.maxDepth, this.limit, this.countOnly,
this.withVertex, this.withPath, this.withEdge);
}
Expand Down
48 changes: 30 additions & 18 deletions hugegraph-api/src/main/java/org/apache/hugegraph/api/traversers/KoutAPI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
import org.apache.hugegraph.traversal.algorithm.HugeTraverser;
import org.apache.hugegraph.traversal.algorithm.KoutTraverser;
import org.apache.hugegraph.traversal.algorithm.records.KoutRecords;
import org.apache.hugegraph.traversal.algorithm.steps.EdgeStep;
import org.apache.hugegraph.traversal.algorithm.steps.Steps;
import org.apache.hugegraph.type.define.Directions;
import org.apache.hugegraph.util.E;
import org.apache.hugegraph.util.Log;
Expand Down Expand Up @@ -126,18 +126,19 @@ public String post(@Context GraphManager manager,
E.checkArgumentNotNull(request, "The request body can't be null");
E.checkArgumentNotNull(request.source,
"The source of request can't be null");
E.checkArgument(request.step != null,
E.checkArgument(request.steps != null,
"The steps of request can't be null");
if (request.countOnly) {
E.checkArgument(!request.withVertex && !request.withPath && !request.withEdge,
"Can't return vertex, edge or path when count only");
}
HugeTraverser.checkTraverseMode(request.traverseMode);

LOG.debug("Graph [{}] get customized kout from source vertex '{}', " +
"with step '{}', max_depth '{}', nearest '{}', " +
"with steps '{}', max_depth '{}', nearest '{}', " +
"count_only '{}', capacity '{}', limit '{}', " +
"with_vertex '{}', with_path '{}' and with_edge '{}'",
graph, request.source, request.step, request.maxDepth,
graph, request.source, request.steps, request.maxDepth,
request.nearest, request.countOnly, request.capacity,
request.limit, request.withVertex, request.withPath,
request.withEdge);
Expand All @@ -147,14 +148,22 @@ public String post(@Context GraphManager manager,
HugeGraph g = graph(manager, graph);
Id sourceId = HugeVertex.getIdValue(request.source);

EdgeStep step = step(g, request.step);
Steps steps = steps(g, request.steps);
KoutRecords results;
try (KoutTraverser traverser = new KoutTraverser(g)) {
results = traverser.customizedKout(sourceId, step,
request.maxDepth,
request.nearest,
request.capacity,
request.limit);
if (HugeTraverser.isTraverseModeDFS(request.traverseMode)) {
results = traverser.dfsKout(sourceId, steps,
request.maxDepth,
request.nearest,
request.capacity,
request.limit);
} else {
results = traverser.customizedKout(sourceId, steps,
request.maxDepth,
request.nearest,
request.capacity,
request.limit);
}
measure.addIterCount(traverser.vertexIterCounter.get(),
traverser.edgeIterCounter.get());
}
Expand All @@ -172,7 +181,7 @@ public String post(@Context GraphManager manager,

if (request.countOnly) {
return manager.serializer(g, measure.measures())
.writeNodesWithPath("kneighbor", neighbors, size, paths,
.writeNodesWithPath("kout", neighbors, size, paths,
QueryResults.emptyIterator(),
QueryResults.emptyIterator());
}
Expand Down Expand Up @@ -210,8 +219,8 @@ private static class Request {

@JsonProperty("source")
public Object source;
@JsonProperty("step")
public TraverserAPI.Step step;
@JsonProperty("steps")
public TraverserAPI.VESteps steps;
@JsonProperty("max_depth")
public int maxDepth;
@JsonProperty("nearest")
Expand All @@ -228,16 +237,19 @@ private static class Request {
public boolean withPath = false;
@JsonProperty("with_edge")
public boolean withEdge = false;
@JsonProperty("traverse_mode")
public String traverseMode = HugeTraverser.TRAVERSE_MODE_BFS;

@Override
public String toString() {
return String.format("KoutRequest{source=%s,step=%s,maxDepth=%s" +
return String.format("KoutRequest{source=%s,steps=%s,maxDepth=%s" +
"nearest=%s,countOnly=%s,capacity=%s," +
"limit=%s,withVertex=%s,withPath=%s," +
"withEdge=%s}", this.source, this.step,
this.maxDepth, this.nearest, this.countOnly,
this.capacity, this.limit, this.withVertex,
this.withPath, this.withEdge);
"withEdge=%s,traverseMode=%s}", this.source,
this.steps, this.maxDepth, this.nearest,
this.countOnly, this.capacity, this.limit,
this.withVertex, this.withPath, this.withEdge,
this.traverseMode);
}
}
}
60 changes: 60 additions & 0 deletions hugegraph-api/src/main/java/org/apache/hugegraph/api/traversers/TraverserAPI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,16 @@

import static org.apache.hugegraph.traversal.algorithm.HugeTraverser.DEFAULT_MAX_DEGREE;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.hugegraph.HugeGraph;
import org.apache.hugegraph.api.API;
import org.apache.hugegraph.traversal.algorithm.steps.EdgeStep;
import org.apache.hugegraph.traversal.algorithm.steps.Steps;
import org.apache.hugegraph.type.define.Directions;

import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonProperty;

Expand All @@ -36,6 +39,25 @@ protected static EdgeStep step(HugeGraph graph, Step step) {
step.maxDegree, step.skipDegree);
}

protected static Steps steps(HugeGraph graph, VESteps steps) {
Map<String, Map<String, Object>> vSteps = new HashMap<>();
if (steps.vSteps != null) {
for (VEStepEntity vStep : steps.vSteps) {
vSteps.put(vStep.label, vStep.properties);
}
}

Map<String, Map<String, Object>> eSteps = new HashMap<>();
if (steps.eSteps != null) {
for (VEStepEntity eStep : steps.eSteps) {
eSteps.put(eStep.label, eStep.properties);
}
}

return new Steps(graph, steps.direction, vSteps, eSteps,
steps.maxDegree, steps.skipDegree);
}

protected static class Step {

@JsonProperty("direction")
Expand All @@ -58,4 +80,42 @@ public String toString() {
this.maxDegree, this.skipDegree);
}
}

protected static class VEStepEntity {

@JsonProperty("label")
public String label;

@JsonProperty("properties")
public Map<String, Object> properties;

@Override
public String toString() {
return String.format("VEStepEntity{label=%s,properties=%s}",
this.label, this.properties);
}
}

protected static class VESteps {

@JsonProperty("direction")
public Directions direction;
@JsonAlias("degree")
@JsonProperty("max_degree")
public long maxDegree = Long.parseLong(DEFAULT_MAX_DEGREE);
@JsonProperty("skip_degree")
public long skipDegree = 0L;
@JsonProperty("vertex_steps")
public List<VEStepEntity> vSteps;
@JsonProperty("edge_steps")
public List<VEStepEntity> eSteps;

@Override
public String toString() {
return String.format("Steps{direction=%s,maxDegree=%s," +
"skipDegree=%s,vSteps=%s,eSteps=%s}",
this.direction, this.maxDegree,
this.skipDegree, this.vSteps, this.eSteps);
}
}
}
1 change: 1 addition & 0 deletions hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/Query.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,7 @@ public boolean reachLimit(long count) {
/**
* Set or update the offset and limit by a range [start, end)
* NOTE: it will use the min range one: max start and min end
*
* @param start the range start, include it
* @param end the range end, exclude it
*/
Expand Down
55 changes: 44 additions & 11 deletions hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -690,7 +690,7 @@ public void removeVertex(HugeVertex vertex) {
// Override vertices in local `addedVertices`
this.addedVertices.remove(vertex.id());
// Force load vertex to ensure all properties are loaded (refer to #2181)
if (vertex.schemaLabel().indexLabels().size() > 0) {
if (vertex.schemaLabel().indexLabels().size() > 0) {
vertex.forceLoad();
}
// Collect the removed vertex
Expand Down Expand Up @@ -937,7 +937,7 @@ protected Iterator<Edge> queryEdgesByIds(Object[] edgeIds,
* local vertex and duplicated id.
*/
Iterator<HugeEdge> it = this.queryEdgesFromBackend(query);
@SuppressWarnings({ "unchecked", "rawtypes" })
@SuppressWarnings({"unchecked", "rawtypes"})
Iterator<Edge> r = (Iterator) it;
return r;
}
Expand Down Expand Up @@ -1195,9 +1195,10 @@ public <V> void removeEdgeProperty(HugeEdgeProperty<V> prop) {
/**
* Construct one edge condition query based on source vertex, direction and
* edge labels
*
* @param sourceVertex source vertex of edge
* @param direction only be "IN", "OUT" or "BOTH"
* @param edgeLabels edge labels of queried edges
* @param direction only be "IN", "OUT" or "BOTH"
* @param edgeLabels edge labels of queried edges
* @return constructed condition query
*/
@Watched
Expand Down Expand Up @@ -1230,8 +1231,39 @@ public static ConditionQuery constructEdgesQuery(Id sourceVertex,
} else if (edgeLabels.length > 1) {
query.query(Condition.in(HugeKeys.LABEL,
Arrays.asList(edgeLabels)));
}

return query;
}

public static ConditionQuery constructEdgesQuery(Id sourceVertex,
Directions direction,
List<Id> edgeLabels) {
E.checkState(sourceVertex != null,
"The edge query must contain source vertex");
E.checkState(direction != null,
"The edge query must contain direction");

ConditionQuery query = new ConditionQuery(HugeType.EDGE);

// Edge source vertex
query.eq(HugeKeys.OWNER_VERTEX, sourceVertex);

// Edge direction
if (direction == Directions.BOTH) {
query.query(Condition.or(
Condition.eq(HugeKeys.DIRECTION, Directions.OUT),
Condition.eq(HugeKeys.DIRECTION, Directions.IN)));
} else {
assert edgeLabels.length == 0;
assert direction == Directions.OUT || direction == Directions.IN;
query.eq(HugeKeys.DIRECTION, direction);
}

// Edge labels
if (edgeLabels.size() == 1) {
query.eq(HugeKeys.LABEL, edgeLabels.get(0));
} else if (edgeLabels.size() > 1) {
query.query(Condition.in(HugeKeys.LABEL, edgeLabels));
}

return query;
Expand Down Expand Up @@ -1363,8 +1395,8 @@ private <R> QueryList<R> optimizeQueries(Query query,
}

boolean supportIn = this.storeFeatures().supportsQueryWithInCondition();
for (ConditionQuery cq: ConditionQueryFlatten.flatten(
(ConditionQuery) query, supportIn)) {
for (ConditionQuery cq : ConditionQueryFlatten.flatten(
(ConditionQuery) query, supportIn)) {
// Optimize by sysprop
Query q = this.optimizeQuery(cq);
/*
Expand All @@ -1387,7 +1419,7 @@ private Query optimizeQuery(ConditionQuery query) {
"Not supported querying by id and conditions: %s", query);
}

Id label = (Id) query.condition(HugeKeys.LABEL);
Id label = query.condition(HugeKeys.LABEL);

// Optimize vertex query
if (label != null && query.resultType().isVertex()) {
Expand Down Expand Up @@ -1580,6 +1612,7 @@ private void checkNonnullProperty(HugeVertex vertex) {
@SuppressWarnings("unchecked")
Collection<Id> missed = CollectionUtils.subtract(nonNullKeys, keys);
HugeGraph graph = this.graph();

E.checkArgument(false, "All non-null property keys %s of " +
"vertex label '%s' must be set, missed keys %s",
graph.mapPkId2Name(nonNullKeys), vertexLabel.name(),
Expand Down Expand Up @@ -1803,9 +1836,9 @@ private Iterator<?> joinTxVertices(Query query,
// Filter vertices matched conditions
return q.test(v) ? v : null;
};
vertices = this.joinTxRecords(query, vertices, matchTxFunc,
this.addedVertices, this.removedVertices,
this.updatedVertices);
vertices = this.joinTxRecords(query, vertices, matchTxFunc,
this.addedVertices, this.removedVertices,
this.updatedVertices);
return vertices;
}

Expand Down
Loading

0 comments on commit 8e99d85

Please sign in to comment.