From fec3050e670b2712b72c9c6bf6cd3b2d30eec8bd Mon Sep 17 00:00:00 2001 From: shirly121 Date: Sun, 29 Sep 2024 19:25:19 +0800 Subject: [PATCH 1/2] [GIE Compiler] add concurrent tests for BI queries --- .../com/alibaba/graphscope/GraphServer.java | 3 + .../common/ir/planner/GraphRelOptimizer.java | 10 +++- .../ir/planner/PlannerGroupManager.java | 18 +++++- .../common/ConcurrentProcessRunner.java | 57 +++++++++++++++++++ .../common/ir/planner/cbo/BITest.java | 8 +++ .../ir/planner/cbo/ConcurrentBITest.java | 27 +++++++++ 6 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ConcurrentProcessRunner.java create mode 100644 interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/ConcurrentBITest.java diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java index d8bafa8cd5bc..4ac59f7a7ad9 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/GraphServer.java @@ -159,6 +159,9 @@ public void close() throws Exception { if (!FrontendConfig.GREMLIN_SERVER_DISABLED.get(configs) && this.gremlinServer != null) { this.gremlinServer.close(); } + if (this.optimizer != null) { + this.optimizer.close(); + } } public static void main(String[] args) throws Exception { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphRelOptimizer.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphRelOptimizer.java index ae3039bd132f..288701e9fb63 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphRelOptimizer.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphRelOptimizer.java @@ -44,13 +44,14 @@ import org.apache.calcite.tools.RelBuilderFactory; import org.checkerframework.checker.nullness.qual.Nullable; +import java.io.Closeable; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; /** * Optimize graph relational tree which consists of match and other relational operators */ -public class GraphRelOptimizer { +public class GraphRelOptimizer implements Closeable { private final PlannerConfig config; private final RelBuilderFactory relBuilderFactory; private final GlogueHolder glogueHolder; @@ -97,6 +98,13 @@ public RelNode optimize(RelNode before, GraphIOProcessor ioProcessor) { return null; } + @Override + public void close() { + if (this.plannerGroupManager != null) { + this.plannerGroupManager.close(); + } + } + public static class MatchOptimizer extends GraphShuttle { private final GraphIOProcessor ioProcessor; private final RelOptPlanner matchPlanner; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/PlannerGroupManager.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/PlannerGroupManager.java index c7f10be17928..0abfb7cb6d9b 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/PlannerGroupManager.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/PlannerGroupManager.java @@ -25,13 +25,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.Closeable; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; -public abstract class PlannerGroupManager { +public abstract class PlannerGroupManager implements Closeable { protected final PlannerConfig config; protected final RelBuilderFactory relBuilderFactory; @@ -40,6 +41,9 @@ public PlannerGroupManager(PlannerConfig config, RelBuilderFactory relBuilderFac this.relBuilderFactory = relBuilderFactory; } + @Override + public void close() {} + public abstract PlannerGroup getCurrentGroup(); public static class Static extends PlannerGroupManager { @@ -103,5 +107,17 @@ public PlannerGroup getCurrentGroup() { int groupId = (int) Thread.currentThread().getId() % plannerGroups.size(); return plannerGroups.get(groupId); } + + @Override + public void close() { + try { + if (this.clearScheduler != null) { + this.clearScheduler.shutdown(); + this.clearScheduler.awaitTermination(10 * 1000, TimeUnit.MILLISECONDS); + } + } catch (Exception e) { + logger.error("failed to close planner group manager.", e); + } + } } } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ConcurrentProcessRunner.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ConcurrentProcessRunner.java new file mode 100644 index 000000000000..7989c8f2296b --- /dev/null +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ConcurrentProcessRunner.java @@ -0,0 +1,57 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common; + +import org.apache.tinkerpop.gremlin.process.GremlinProcessRunner; +import org.junit.runners.model.InitializationError; +import org.junit.runners.model.RunnerScheduler; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +/** + * A JUnit runner that runs tests concurrently, the default thread pool size is 4. + */ +public class ConcurrentProcessRunner extends GremlinProcessRunner { + private final ExecutorService executorService; + + public ConcurrentProcessRunner(Class klass) throws InitializationError { + super(klass); + this.executorService = Executors.newFixedThreadPool(4); + RunnerScheduler scheduler = + new RunnerScheduler() { + @Override + public void schedule(Runnable childStatement) { + executorService.submit(childStatement); + } + + @Override + public void finished() { + try { + executorService.shutdown(); + executorService.awaitTermination(120, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + }; + setScheduler(scheduler); + } +} diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/BITest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/BITest.java index bd33203fb90c..788d5e19909f 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/BITest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/BITest.java @@ -27,6 +27,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.calcite.rel.RelNode; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -56,6 +57,13 @@ public static void beforeClass() { optimizer.getGlogueHolder()); } + @AfterClass + public static void afterClass() { + if (optimizer != null) { + optimizer.close(); + } + } + @Test public void bi1_test() { GraphBuilder builder = Utils.mockGraphBuilder(optimizer, irMeta); diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/ConcurrentBITest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/ConcurrentBITest.java new file mode 100644 index 000000000000..1ab6b1289dce --- /dev/null +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/ConcurrentBITest.java @@ -0,0 +1,27 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.common.ir.planner.cbo; + +import com.alibaba.graphscope.common.ConcurrentProcessRunner; + +import org.junit.runner.RunWith; + +// run BI Tests in parallel, to check thread-safety of the CBO optimization +@RunWith(ConcurrentProcessRunner.class) +public class ConcurrentBITest extends BITest {} From 8301b71a60ffb6a960e1890da4ea5e6e21983d0a Mon Sep 17 00:00:00 2001 From: shirly121 Date: Tue, 1 Oct 2024 22:10:47 +0800 Subject: [PATCH 2/2] minor fix --- .../graphscope/common/ir/planner/cbo/ConcurrentBITest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/ConcurrentBITest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/ConcurrentBITest.java index 1ab6b1289dce..54dbc8ae0581 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/ConcurrentBITest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/ConcurrentBITest.java @@ -23,5 +23,6 @@ import org.junit.runner.RunWith; // run BI Tests in parallel, to check thread-safety of the CBO optimization +// ConcurrentProcessRunner is a custom runner that runs tests in parallel @RunWith(ConcurrentProcessRunner.class) public class ConcurrentBITest extends BITest {}