diff --git a/.travis.yml b/.travis.yml index 5b6b80fff..2e5317e8a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,8 @@ language: - scala sudo: false before_script: -- mkdir -p $HOME/.sbt/launchers/0.13.9/ -- curl -L -o $HOME/.sbt/launchers/0.13.9/sbt-launch.jar http://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/0.13.9/sbt-launch.jar +- mkdir -p $HOME/.sbt/launchers/0.13.11/ +- curl -L -o $HOME/.sbt/launchers/0.13.11/sbt-launch.jar http://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/0.13.11/sbt-launch.jar script: - echo "TRAVIS_PULL_REQUEST" $TRAVIS_PULL_REQUEST - echo "TRAVIS_BRANCH" $TRAVIS_BRANCH @@ -12,14 +12,18 @@ script: - echo "repo" $TRAVIS_REPO_SLUG - set -o pipefail; skipLogs="Resolving |Compiling |Done updating|Updating |scoverage|coverage-report"; if [[ $TRAVIS_PULL_REQUEST != "false" || $TRAVIS_REPO_SLUG != "gearpump/gearpump" ]]; - then sbt -jvm-opts project/travis/jvmopts clean coverage +test | grep -v -E "$skipLogs"; + then sbt -jvm-opts project/travis/jvmopts clean scalastyle test:scalastyle it:scalastyle unidoc coverage +test | grep -v -E "$skipLogs"; elif [[ $TRAVIS_BRANCH == "master" ]]; - then sbt -jvm-opts project/travis/jvmopts clean +publish | grep -v -E "$skipLogs"; sbt -jvm-opts project/travis/jvmopts coverage +test | grep -v -E "$skipLogs"; + then sbt -jvm-opts project/travis/jvmopts clean +publish | grep -v -E "$skipLogs"; sbt -jvm-opts project/travis/jvmopts scalastyle test:scalastyle it:scalastyle unidoc coverage +test | grep -v -E "$skipLogs"; elif [[ $TRAVIS_TAG != "" ]]; then sbt -jvm-opts project/travis/jvmopts clean +assembly +packArchiveZip | grep -v -E "$skipLogs"; fi jdk: - oraclejdk8 +addons: + apt: + packages: + - oracle-java8-installer scala: - 2.11.8 cache: diff --git a/CHANGELOG.md b/CHANGELOG.md index 230ec66d7..a5681fc4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ Mail list: What are the changes for the process for Apache Incubation? ------------------ -The code importing to Apache Git is still in process. During this transtion time, +The code importing to Apache Git is still in process. During this transition time, We will follow these procedures: 1. For new issue, we should no longer open issue in Github issues. Instead, we @@ -43,7 +43,6 @@ After the transition period, the proposed process is like this: Before completing importing source code to Apache Git, we will still use - Why we make a tag release now, but not wait until Apache importing complete? ------------------ There are quite a lot of open task at https://issues.apache.org/jira/browse/GEARPUMP-1, which @@ -92,7 +91,7 @@ From now on, new issues should be submitted to https://issues.apache.org/jira/br - #1892: added instruction text to operate network graph (2) minor tweaks of context menu - #1963, rename CommitGuideline.md to CONTRIBUTING.md - #1958: better test report organizing -- #1906: some visual glichtes found in mobile view +- #1906: some visual glitches found in mobile view Gearpump 0.7.6 =================== @@ -232,13 +231,13 @@ Change logs: - #1701: wrong dirty check of modify processor and more - #1384: rest service url was not set correctly - #1682: allow specify a transition time when changing parallelism -- #1691: dashboard layout and textual changes and update to dashing… -- #1640 YARN deployment - no easy way of obtaining master address… +- #1691: dashboard layout and textual changes and update to dashing... +- #1640 YARN deployment - no easy way of obtaining master address... - #1688, Reduce unimportant logs in travis UT console - #1671, make default timeout shorter to be more UI responsive. - #1685, Should not publish gearpump-integration-test to maven - #1683, Allow metrics to be aggregated before sending to UI -- #1223: will mark application with a warning when stalling task de… +- #1223: will mark application with a warning when stalling task de... - #1670, #1653, fix dynamic dag bug - #1659 add a rest api to terminate service process - #1672: when worker is killed, its detail page will no longer be updated @@ -251,7 +250,7 @@ Change logs: - #1639 refine daemon's classpath - #1652, submit app with no arguments from dashboard returns warning - #1547 add a SequenceFileSink -- #1424: when application is terminated, related status indicators … +- #1424: when application is terminated, related status indicators ... - #1645: parse appid in submission response better - #1514, add an experimental module to support akka-stream 1.0 - #1643, fix UI timeout issue when metrics is disabled @@ -322,9 +321,9 @@ Change log: - #1194 add graph cycle detection - #491, add a rest api to acquire Gearpump's built-in partitioners - #1405: now the number of executors is back -- #1371: a right way to show historical metrics (2) added committed… +- #1371: a right way to show historical metrics (2) added committed? - #1402, fix NoneGroupingPartitioner -- #1399: in application overview page the task count of executor wa… +- #1399: in application overview page the task count of executor wa... - #1397, allow KafkaSink to receive Message of bytes - #1395, cross publishSigned - #1374 remove jars of yarn in executor's classpath @@ -337,15 +336,13 @@ Gearpump 0.6.1 ========================== Highlight: new version UI dashboard. - Change log: -------------------- - #1369: object path and metrics name are divided by ':' now - #1369: fixed data initialization issue of echart - #1364, add default value for BYTES_PER_MESSAGE of SOLStreamProducer -- #1361: able to show multiple tasks in chart (better not select mo… -- #1358: properties will be updated (2) moved roboto font and echar… - +- #1361: able to show multiple tasks in chart (better not select mo? +- #1358: properties will be updated (2) moved roboto font and echart? Gearpump 0.6 ========================== @@ -357,7 +354,6 @@ Change log: - #1352, return appmaster config - #1344 fix Master HA bug - Gearpump 0.5.0 ========================== Highlights: @@ -466,7 +462,7 @@ Change logs: - #1149: Shell tools printed too much detail on console - #1146: actor hungry when worker use block-io to wait response from FileServer. - #1088: move hbase code to external/ - - #1140：pass app name to task + - #1140: pass app name to task - #1017: Split daemon dependencies with core dependencies - #1144: fix out of memory when trying to scale gearpump to 2000 task on a 2 core machine - #995: Command line parser should be able to fall back to MAIN-CLASS definition in MANIFEST.IN when mainClass is not specified in command line options. @@ -540,7 +536,7 @@ Change list - #775, fix netty config - #778, log improvements - #781 when launching lots of tasks, the application failed to transfer message across hosts - - #782, a) add wildcard match to return metrics(support glob char … and *), b) add diagnosis message if the clock stop advancing + - #782, a) add wildcard match to return metrics(support glob char . and *), b) add diagnosis message if the clock stop advancing - #786, Read user config from classpath, the appmaster/executor wil use the dynamic user config - #773: skew chart will show relative skew - #790, 1) return detail task data in appmaster REST. 2) bind executor id with executor system id @@ -567,8 +563,8 @@ Change list - #843, Can't put custom user config in application.conf - #849, set default hostname to 127.0.0.1 in UT - #851, JVM not exited when there is exception due to akka create non-daemon threads - - #854， fix storm connector performance - - #856， Service launch failed + - #854, fix storm connector performance + - #856, Service launch failed - #853, fix resource leak(thread not closed, process not killed in UT. Also increase the PermGen size to avoid Permgen OOM. - #859, random UT fail due to akka bug, "akka cluster deadlock when initializing" - #865, Change the default timeout setting in akka test expectMsg @@ -595,7 +591,7 @@ gearpump-0.3.4 ==================== Change List: ---------------- - - #768, Serious performance degration of ui server on windows + - #768, Serious performance degrade of ui server on windows - #765, improve the graph type inference so we don't need to set template argument type explicitly gearpump-0.3.3 @@ -718,7 +714,7 @@ Change List: - #479, publish test jar as artifacts - #419, Reorder the application log by master startup timestamp - #456, Use visdag to render the graph DAG - - #464， Travis bower integration + - #464, Travis bower integration - #394, fix kafka producer hang issue - #468, For test code, the ClockService will throw exception when the DAG is not defined - #465, fix appname prefix bug @@ -751,13 +747,12 @@ Change List: - #378, construct the serializer explicitly, instead of implicitly via the kryo serializer akka extension - #380, set the context class loader as the URL class loader for ActorSystem. - gearpump-0.2.3 ==================== Change List: --------------- - #333, KafkaUtilSpec causes out of memory on travis - - #335, #359， Enable auto-deployment to sonatype + - #335, #359, Enable auto-deployment to sonatype - #299, Some UT may fail randomly, most because of the expectMsg time out - #338, fix kafka leader not available exception - #349, scoverage dependencies get into snapshot build binaries. @@ -797,7 +792,7 @@ Highlights: - Add support for general replay-able data source to support at least once delivery. - More robust streaming. Be resilient to message loss, message duplication, and zombie processes. - Refactor Kafka data source for at least once delivery. - - Support general applications besides streaming, add an experimental distrubtedshell application under experiments/. + - Support general applications besides streaming, add an experimental distributedshell application under experiments/. - Re-defined the AppMaster, and Task interface, It is much easier to write an application now. - Support submitting and distributing large applications jars. - Add CI tool, nightly build, code coverage, and defined a formal commit guideline. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a523344ef..a39da53cb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,7 +35,7 @@ No work should ever be done in the forked master. Another way to do this is to ``` git branch branch_issueId - git checkout branch_issusId + git checkout branch_issueId ``` Work on the branch, make changes, then push to your forked gearpump repo http://github.com//gearpump @@ -78,7 +78,6 @@ No work should ever be done in the forked master. Another way to do this is to 6. Open a PR, which is a one-click thing in github.com; it knows you likely are opening a PR against upstream master. [Guide](https://help.github.com/articles/creating-a-pull-request) is here. 7. [Merge PR](https://help.github.com/articles/merging-a-pull-request), [delete branch](https://help.github.com/articles/deleting-unused-branches). - You can skip using branches in your fork if desired, and just work out of your master. Less overhead, but, you can't pursue different unrelated changes then. The key here is using rebase, not merge. The downstream forks never need to use merge unless the work is a long-lived collaboration with someone else who is pulling from your fork. diff --git a/LICENSE b/LICENSE index 106b49b66..97a5197c2 100644 --- a/LICENSE +++ b/LICENSE @@ -200,7 +200,6 @@ See the License for the specific language governing permissions and limitations under the License. - ----------------------------------------------------------------------- For bootstrap 3.3.5/3.3.6 @@ -300,13 +299,11 @@ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - ----------------------------------------------------------------------- For angular 1.5.0 (angular-1.5.0-rc.0.jar) https://github.com/angular/angular.js/blob/master/LICENSE - The MIT License Copyright (c) 2010-2016 Google, Inc. http://angularjs.org @@ -329,7 +326,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- For angular-loading-bar 0.8.0 (angular-loading-bar-0.8.0.jar) @@ -371,7 +367,6 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- For angular-smart-table-2.1.6 (angular-smart-table-2.1.6.jar) @@ -387,7 +382,6 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- For angular-strap-2.3.5 (angular-strap-2.3.5.jar) @@ -403,7 +397,6 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- For angular-ui-router-0.2.15 (angular-ui-router-0.2.15.jar) @@ -431,7 +424,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- For angular-ui-select-0.13.2.jar (angular-ui-select-0.13.2.jar) @@ -458,7 +450,6 @@ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- For angularjs-1.4.8(angularjs-1.4.8.jar) @@ -486,7 +477,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- For aopalliance-1.0 (aopalliance-1.0.jar) @@ -501,7 +491,6 @@ LICENCE: all the source code provided by AOP Alliance is Public Domain. For asm-3.1 (asm-3.1.jar) http://asm.ow2.org/license.html - Copyright (c) 2000-2011 INRIA, France Telecom All rights reserved. @@ -537,7 +526,6 @@ THE POSSIBILITY OF SUCH DAMAGE. For bootstrap-additions-0.3.1 (bootstrap-additions-0.3.1.jar) https://github.com/twbs/bootstrap/blob/master/LICENSE - The MIT License Copyright (c) 2014 Olivier Louvignes http://olouv.com @@ -548,7 +536,6 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- For clipboard.js-0.1.1 (clipboard.js-0.1.1.jar) @@ -587,7 +574,6 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- For font-awesome-4.5.0 (font-awesome-4.5.0.jar) @@ -614,7 +600,6 @@ All code is available to you under the MIT license, available at http://opensour Copyright Erik Osheim, 2012-2015. - ----------------------------------------------------------------------- For jaxb 2.2.2, jaxb 2.2.3 (jaxb-api-2.2.2.jar, jaxb-impl-2.2.3-1.jar) @@ -633,7 +618,6 @@ COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL - Version 1.1) GNU General Public License (GPL - Version 2, June 1991) with the [Classpath Exception] - ----------------------------------------------------------------------- For jgrapht-core-0.9.0 (jgrapht-core-0.9.0.jar) @@ -907,7 +891,6 @@ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ----------------------------------------------------------------------- For jquery-cookie-1.4.1 (jquery-cookie-1.4.1.jar) @@ -971,7 +954,6 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ----------------------------------------------------------------------- For jsr305-3.0.0 (jsr305-3.0.0.jar) @@ -1006,13 +988,11 @@ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ----------------------------------------------------------------------- For kryo-2.21.jar https://code.google.com/p/kryo/ - Code license New BSD License @@ -1021,7 +1001,6 @@ New BSD License For leveldbjni-all-1.8.jar https://github.com/fusesource/leveldbjni - Copyright (c) 2011 FuseSource Corp. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -1111,7 +1090,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- For ng-file-upload-5.0.9.jar @@ -1138,7 +1116,6 @@ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- For leveldbjniparanamer-2.3.jar @@ -1280,7 +1257,6 @@ License API is licensed under standard BSD license, which is compatible with all Free and Open Software (OSS) licenses. - ----------------------------------------------------------------------- For upickle_2.11-0.3.4.jar @@ -1297,13 +1273,11 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ----------------------------------------------------------------------- For xmlenc-0.52.jar https://github.com/digidentity/xmlenc/blob/master/LICENSE.txt - Copyright (c) 2013 Benoist MIT License diff --git a/README.md b/README.md index 32b3837c4..17f0a0f2b 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,14 @@ export HTTPS_PROXY= http://host:port ## How to run Gearpump integration test Gearpump has an integration test system which is based on Docker. Please check [the instructions](integrationtest/README.md). +## How to do style check before submitting a pull request? + +Before submitting a PR, you should always run style check first: +``` + ## Run style check for compile, test, and integration test. + sbt scalastyle test:scalastyle it:scalastyle +``` + ## Contributors (time order) * [Sean Zhong](https://github.com/clockfly) diff --git a/ReleaseProcess.md b/ReleaseProcess.md index 7bf9e3c3d..bc035c35b 100644 --- a/ReleaseProcess.md +++ b/ReleaseProcess.md @@ -38,5 +38,4 @@ Step3: Post-Release where NEXT_SNPASHOT_VERSION must end with "-SNAPSHOT". For example, 0.2.3-SNPASHOT is a good snapshot version, 0.2.3 is NOT 2. Make PR by following https://github.com/gearpump/gearpump/blob/master/doc/CommitGuideline.md - Done!!! diff --git a/conf/gear.conf b/conf/gear.conf index c2352fed9..b21f59ac0 100644 --- a/conf/gear.conf +++ b/conf/gear.conf @@ -166,8 +166,6 @@ gearpump { # "yourpackage.YourClass" = "" } - - ### client's timeout (in second) to connect to master and wait for the response masterclient.timeout = 90 @@ -316,7 +314,6 @@ gearpump { } } - ### Configuration only visible to master nodes.. gearpump-master { extensions = [ @@ -369,13 +366,11 @@ gearpump-worker { } } - ######################################### ### For log level of Akka class, you need to change both log4j.properties and this entry ######################################### #akka.loglevel = "INFO" - ### configurations only visible to UI server. gearpump-ui { ## Security related settings @@ -514,7 +509,6 @@ gearpump-ui { ## value set here serves as an example. uaahost = "http://" - ## Whether to enable additional authorization check. ## If the user fails the check, then Gearpump would log user out. additional-authenticator-enabled = true diff --git a/conf/log4j.properties b/conf/log4j.properties index a6013ca6a..71757786b 100644 --- a/conf/log4j.properties +++ b/conf/log4j.properties @@ -7,7 +7,7 @@ # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -61,7 +61,6 @@ log4j.rootLogger=${log4j.rootLevel},${log4j.rootAppender} # Logging Threshold log4j.threshhold=ALL - # ===================================================================== # Customize logger for a specific class # For example, if you want to enable debug log for specific akka class @@ -88,7 +87,6 @@ log4j.appender.RollingFileAppender.layout.ConversionPattern=%d{ISO8601} %p %c{1} log4j.appender.RollingFileAppender.MaxFileSize=200MB log4j.appender.RollingFileAppender.MaxBackupIndex=10 - # # console appender # Add "console" to rootlogger above if you want to use this @@ -110,4 +108,3 @@ log4j.appender.ApplicationLogAppender.MaxBackupIndex=30 log4j.appender.ApplicationLogAppender.layout=org.apache.log4j.PatternLayout log4j.appender.ApplicationLogAppender.layout.ConversionPattern=[%p] [%d{MM/dd/yyyy HH:mm:ss.SSS}] [%c{1}] %m%n - diff --git a/core/src/main/java/io/gearpump/transport/netty/ITransportMessageSerializer.java b/core/src/main/java/io/gearpump/transport/netty/ITransportMessageSerializer.java index 9b0734a23..c01135ea8 100644 --- a/core/src/main/java/io/gearpump/transport/netty/ITransportMessageSerializer.java +++ b/core/src/main/java/io/gearpump/transport/netty/ITransportMessageSerializer.java @@ -15,12 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package io.gearpump.transport.netty; import java.io.DataInput; import java.io.DataOutput; public interface ITransportMessageSerializer { + int getLength(Object obj); void serialize(DataOutput dataOutput, Object transportMessage); diff --git a/core/src/main/java/io/gearpump/transport/netty/MessageBatch.java b/core/src/main/java/io/gearpump/transport/netty/MessageBatch.java index 665c99fc4..f55616177 100644 --- a/core/src/main/java/io/gearpump/transport/netty/MessageBatch.java +++ b/core/src/main/java/io/gearpump/transport/netty/MessageBatch.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -29,6 +29,9 @@ import java.util.ArrayList; import java.util.List; +/** + * Netty message on the wire is wrapped as MessageBatch + */ public class MessageBatch { private static final Logger log = LoggerFactory.getLogger(MessageBatch.class); @@ -83,7 +86,6 @@ private int msgEncodeLength(TaskMessage taskMsg) { } /** - * * @return true, if allowed buffer is Full */ boolean isFull() { @@ -91,7 +93,6 @@ boolean isFull() { } /** - * * @return true, if no messages in this batch */ boolean isEmpty() { @@ -99,7 +100,6 @@ boolean isEmpty() { } /** - * * @return number of messages available in this batch */ int size() { @@ -111,7 +111,7 @@ int size() { */ ChannelBuffer buffer() throws IOException { ChannelBufferOutputStream bout = - new ChannelBufferOutputStream(ChannelBuffers.directBuffer(encoded_length)); + new ChannelBufferOutputStream(ChannelBuffers.directBuffer(encoded_length)); try { for (TaskMessage msg : messages) { @@ -128,7 +128,7 @@ ChannelBuffer buffer() throws IOException { /** * write a TaskMessage into a stream - *

+ *

* Each TaskMessage is encoded as: * sessionId ... int(4) * source task ... Long(8) @@ -137,7 +137,7 @@ ChannelBuffer buffer() throws IOException { * payload ... byte[] * */ private void writeTaskMessage(ChannelBufferOutputStream bout, - TaskMessage message) throws IOException { + TaskMessage message) throws IOException { long target_id = message.targetTask(); long source_id = message.sourceTask(); int sessionId = message.sessionId(); diff --git a/core/src/main/java/io/gearpump/transport/netty/MessageDecoder.java b/core/src/main/java/io/gearpump/transport/netty/MessageDecoder.java index d9a74b1f9..7566014ac 100644 --- a/core/src/main/java/io/gearpump/transport/netty/MessageDecoder.java +++ b/core/src/main/java/io/gearpump/transport/netty/MessageDecoder.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -23,7 +23,6 @@ import org.jboss.netty.channel.ChannelHandlerContext; import org.jboss.netty.handler.codec.frame.FrameDecoder; -import java.io.DataInput; import java.util.ArrayList; import java.util.List; @@ -31,7 +30,7 @@ public class MessageDecoder extends FrameDecoder { private ITransportMessageSerializer serializer; private WrappedChannelBuffer dataInput = new WrappedChannelBuffer(); - public MessageDecoder(ITransportMessageSerializer serializer){ + public MessageDecoder(ITransportMessageSerializer serializer) { this.serializer = serializer; } @@ -44,7 +43,7 @@ public MessageDecoder(ITransportMessageSerializer serializer){ * payload ... byte[] * */ protected List decode(ChannelHandlerContext ctx, Channel channel, - ChannelBuffer buf) { + ChannelBuffer buf) { this.dataInput.setChannelBuffer(buf); final int SESION_LENGTH = 4; //int diff --git a/core/src/main/java/io/gearpump/transport/netty/MessageEncoder.java b/core/src/main/java/io/gearpump/transport/netty/MessageEncoder.java index 5bd512d47..674e96fdb 100644 --- a/core/src/main/java/io/gearpump/transport/netty/MessageEncoder.java +++ b/core/src/main/java/io/gearpump/transport/netty/MessageEncoder.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -26,7 +26,7 @@ public class MessageEncoder extends OneToOneEncoder { @Override protected Object encode(ChannelHandlerContext ctx, Channel channel, Object obj) throws Exception { if (obj instanceof MessageBatch) { - return ((MessageBatch)obj).buffer(); + return ((MessageBatch) obj).buffer(); } throw new RuntimeException("Unsupported encoding of object of class " + obj.getClass().getName()); diff --git a/core/src/main/java/io/gearpump/transport/netty/NettyRenameThreadFactory.java b/core/src/main/java/io/gearpump/transport/netty/NettyRenameThreadFactory.java index f6f234c6a..026684e4a 100644 --- a/core/src/main/java/io/gearpump/transport/netty/NettyRenameThreadFactory.java +++ b/core/src/main/java/io/gearpump/transport/netty/NettyRenameThreadFactory.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -38,7 +38,7 @@ public class NettyRenameThreadFactory implements ThreadFactory { NettyRenameThreadFactory(String name) { SecurityManager s = System.getSecurityManager(); group = (s != null) ? s.getThreadGroup() : - Thread.currentThread().getThreadGroup(); + Thread.currentThread().getThreadGroup(); this.name = name; } diff --git a/core/src/main/java/io/gearpump/transport/netty/TaskMessage.java b/core/src/main/java/io/gearpump/transport/netty/TaskMessage.java index 5a5f43c94..243d3f0d4 100644 --- a/core/src/main/java/io/gearpump/transport/netty/TaskMessage.java +++ b/core/src/main/java/io/gearpump/transport/netty/TaskMessage.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -38,13 +38,15 @@ public TaskMessage(int sessionId, long targetTask, long sourceTask, Object messa _message = message; } - public int sessionId() {return _sessionId; } + public int sessionId() { + return _sessionId; + } public long targetTask() { return _targetTask; } - public long sourceTask(){ + public long sourceTask() { return _sourceTask; } diff --git a/core/src/main/java/io/gearpump/transport/netty/WrappedChannelBuffer.java b/core/src/main/java/io/gearpump/transport/netty/WrappedChannelBuffer.java index a89d10395..3531f3bfd 100644 --- a/core/src/main/java/io/gearpump/transport/netty/WrappedChannelBuffer.java +++ b/core/src/main/java/io/gearpump/transport/netty/WrappedChannelBuffer.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package io.gearpump.transport.netty; import org.jboss.netty.buffer.ChannelBuffer; @@ -22,16 +23,18 @@ import java.io.DataInput; import java.io.IOException; -public class WrappedChannelBuffer implements DataInput{ +/** Wrap ChannelBuffer as a DataInput */ +public class WrappedChannelBuffer implements DataInput { private ChannelBuffer channelBuffer; - public WrappedChannelBuffer(){} + public WrappedChannelBuffer() { + } - public WrappedChannelBuffer(ChannelBuffer channelBuffer){ + public WrappedChannelBuffer(ChannelBuffer channelBuffer) { this.channelBuffer = channelBuffer; } - public void setChannelBuffer(ChannelBuffer channelBuffer){ + public void setChannelBuffer(ChannelBuffer channelBuffer) { this.channelBuffer = channelBuffer; } diff --git a/core/src/main/java/io/gearpump/util/AkkaHelper.java b/core/src/main/java/io/gearpump/util/AkkaHelper.java index f4772c18a..286fabf40 100644 --- a/core/src/main/java/io/gearpump/util/AkkaHelper.java +++ b/core/src/main/java/io/gearpump/util/AkkaHelper.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -28,7 +28,12 @@ public class AkkaHelper { * * This is used for performance optimization, we encode the session Id * in the ActorRef path. Session Id is used to identity sender Task. + * + * @param system ActorSystem + * @param path Relative or absolute path of this Actor System. + * @return Full qualified ActorRef. */ + @SuppressWarnings("deprecation") public static ActorRef actorFor(ActorSystem system, String path) { return system.actorFor(path); } diff --git a/core/src/main/java/io/gearpump/util/HadoopFSLogAppender.java b/core/src/main/java/io/gearpump/util/HadoopFSLogAppender.java index ab7ee5e13..93b65e821 100644 --- a/core/src/main/java/io/gearpump/util/HadoopFSLogAppender.java +++ b/core/src/main/java/io/gearpump/util/HadoopFSLogAppender.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/core/src/main/java/io/gearpump/util/RecreateRollingFileAppender.java b/core/src/main/java/io/gearpump/util/RecreateRollingFileAppender.java index 7447eb461..acbd1cba5 100644 --- a/core/src/main/java/io/gearpump/util/RecreateRollingFileAppender.java +++ b/core/src/main/java/io/gearpump/util/RecreateRollingFileAppender.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -23,41 +23,42 @@ import java.io.File; +/** When log file is deleted, tried to recreate it in file system */ public class RecreateRollingFileAppender extends RollingFileAppender { - protected long checkFileInterval = 60L; - private long lastCheckTime = 0L; + protected long checkFileInterval = 60L; + private long lastCheckTime = 0L; - @Override - public void append(LoggingEvent event) { - checkInterval(); - super.append(event); - } + @Override + public void append(LoggingEvent event) { + checkInterval(); + super.append(event); + } - private void checkInterval() { - long currentTime = System.currentTimeMillis(); - if ((currentTime - lastCheckTime) > (checkFileInterval * 1000)) { - checkLogFileExist(); - lastCheckTime = currentTime; - } + private void checkInterval() { + long currentTime = System.currentTimeMillis(); + if ((currentTime - lastCheckTime) > (checkFileInterval * 1000)) { + checkLogFileExist(); + lastCheckTime = currentTime; } + } - private void checkLogFileExist() { - String fileName = super.fileName; - if (fileName != null) { - File logFile = new File(fileName); - if (!logFile.exists()) { - this.setFile(fileName); - this.activateOptions(); - } - } + private void checkLogFileExist() { + String fileName = super.fileName; + if (fileName != null) { + File logFile = new File(fileName); + if (!logFile.exists()) { + this.setFile(fileName); + this.activateOptions(); + } } + } - public long getCheckFileInterval() { - return this.checkFileInterval; - } + public long getCheckFileInterval() { + return this.checkFileInterval; + } - public void setCheckFileInterval(long checkFileInterval) { - this.checkFileInterval = checkFileInterval; - } + public void setCheckFileInterval(long checkFileInterval) { + this.checkFileInterval = checkFileInterval; + } } diff --git a/core/src/main/resources/geardefault.conf b/core/src/main/resources/geardefault.conf index d34a656c1..4eecf3a9f 100644 --- a/core/src/main/resources/geardefault.conf +++ b/core/src/main/resources/geardefault.conf @@ -27,7 +27,6 @@ gearpump { ### hostname = "127.0.0.1" - ## The installation folder of gearpump home = "" @@ -43,7 +42,6 @@ gearpump { ## Number of executors to launch when starting an application application.executor-num = 1 - ## Unique Id to identity this worker instance in low level resource manager like YARN. ## ## This value is typically configured by resource manager integration module, like gearpump-yarn in this case. @@ -86,7 +84,7 @@ gearpump { ### When the resource cannot be allocated in the timeout, then ### the appmaster will shutdown itself. resource-allocation-timeout-seconds = 120 - + ## ## Executor share same process of worker worker.executor-share-same-jvm-as-worker = false @@ -216,8 +214,6 @@ gearpump { "akka.actor.LocalActorRef" = "" } - - ### client's timeout (in second) to connect to master and wait for the response masterclient.timeout = 90 @@ -366,7 +362,6 @@ gearpump { } } - ### Configuration only visible to master nodes.. gearpump-master { ## NOTE: Please add akka-related settings in gear.conf to avoid confusion in @@ -379,13 +374,11 @@ gearpump-worker { ## config overriding. } - ######################################### ### For log level of Akka class, you need to change both log4j.properties and this entry ######################################### #akka.loglevel = "INFO" - ### configurations only visible to UI server. gearpump-ui { ## Security related settings @@ -542,7 +535,6 @@ gearpump-ui { } } - ################################################################### ################################################################### ## The following configurations supercede the default akka config @@ -552,7 +544,7 @@ gearpump-ui { ## work-around to fix akka io performance issue on windows ## reference: http://stackoverflow.com/questions/29453848/high-cpu-on-akka-io-pinned-dispatcher-on-windows ## ticket: https://github.com/akka/akka/issues/16295 -akka.io.tcp.windows-connection-abort-workaround-enabled=off +akka.io.tcp.windows-connection-abort-workaround-enabled = off ### On windows, the value must be larger than 10ms, check ### https://github.com/akka/akka/blob/master/akka-actor/src/main/scala/akka/actor/Scheduler.scala#L204 @@ -576,22 +568,20 @@ akka { ## Akka-http session related settings session { - serverSecret = "!!!please change this to a value only you know!!!" - - clientSession { - cookie { - name = "gearpump_token" - ## domain = "..." - path = "/" - ## maxAge = 0 - secure = false - httpOnly = true - } + server-secret = "Please change this to a value only you know! At least 64 chars!!!" - ## Session lifetime. Default value is about 1 week - maxAgeSeconds = 604800 - encryptData = true + cookie { + name = "gearpump_token" + ## domain = "..." + path = "/" + ## maxAge = 0 + secure = false + http-only = true } + + ## Session lifetime. Default value is about 1 week + max-age = 604800 + encrypt-data = true } } @@ -628,7 +618,6 @@ akka { mailbox-type = "akka.dispatch.SingleConsumerOnlyUnboundedMailbox" } default-dispatcher { - mailbox-type = "akka.dispatch.SingleConsumerOnlyUnboundedMailbox" throughput = 10 fork-join-executor { parallelism-factor = 2 @@ -668,13 +657,11 @@ akka { default-remote-dispatcher { throughput = 5 type = Dispatcher - mailbox-type = "akka.dispatch.SingleConsumerOnlyUnboundedMailbox" executor = "fork-join-executor" } } } - ## Configurations only visible on Linux or Mac. gearpump-linux { ### On windows, the value must be larger than 10ms, check diff --git a/core/src/main/resources/log4j.properties b/core/src/main/resources/log4j.properties index 937905acf..682a11bf3 100644 --- a/core/src/main/resources/log4j.properties +++ b/core/src/main/resources/log4j.properties @@ -7,7 +7,7 @@ # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -54,7 +54,6 @@ log4j.rootLogger=INFO,console # Logging Threshold log4j.threshhold=ALL - # ===================================================================== # Appenders # ===================================================================== @@ -75,7 +74,6 @@ log4j.appender.RollingFileAppender.layout.ConversionPattern=%d{ISO8601} %p %c{1} log4j.appender.RollingFileAppender.MaxFileSize=200MB log4j.appender.RollingFileAppender.MaxBackupIndex=10 - # # console appender # Add "console" to rootlogger above if you want to use this @@ -100,4 +98,3 @@ log4j.appender.ApplicationLogAppender.layout=org.apache.log4j.PatternLayout #log4j.appender.ApplicationLogAppender.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n log4j.appender.ApplicationLogAppender.layout.ConversionPattern=[%p] [%d{MM/dd/yyyy HH:mm:ss.SSS}] [%c{1}] %m%n - diff --git a/core/src/main/scala/io/gearpump/Message.scala b/core/src/main/scala/io/gearpump/Message.scala index b442e336a..aec808954 100644 --- a/core/src/main/scala/io/gearpump/Message.scala +++ b/core/src/main/scala/io/gearpump/Message.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -22,11 +22,11 @@ package io.gearpump * Each message contains a immutable timestamp. * * For example, if you take a picture, the time you take the picture is the - * message's timestmap. - * @param msg Accept any type except Null, Nothing and Unit + * message's timestamp. + * @param msg Accept any type except Null, Nothing and Unit */ case class Message(msg: Any, timestamp: TimeStamp = Message.noTimeStamp) object Message { - val noTimeStamp : TimeStamp = 0L + val noTimeStamp: TimeStamp = 0L } diff --git a/core/src/main/scala/io/gearpump/cluster/AppDescription.scala b/core/src/main/scala/io/gearpump/cluster/AppDescription.scala index e8af8545e..799c20aec 100644 --- a/core/src/main/scala/io/gearpump/cluster/AppDescription.scala +++ b/core/src/main/scala/io/gearpump/cluster/AppDescription.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,44 +18,64 @@ package io.gearpump.cluster +import scala.reflect.ClassTag + import akka.actor.{Actor, ActorRef, ActorSystem} import com.typesafe.config.{Config, ConfigFactory} + import io.gearpump.cluster.appmaster.WorkerInfo import io.gearpump.cluster.scheduler.Resource import io.gearpump.jarstore.FilePath -import io.gearpump.util.Util - -import scala.reflect.ClassTag /** * This contains all information to run an application * - * @param name: The name of this application - * @param appMaster: The class name of AppMaster Actor - * @param userConfig: user configuration. - * @param clusterConfig: The user provided cluster config, it will override gear.conf when starting - * new applications. In most cases, you wouldnot need to change it. If you do need to change it, - * use ClusterConfigSource(filePath) to construct the object, while filePath points to the .conf file. + * @param name The name of this application + * @param appMaster The class name of AppMaster Actor + * @param userConfig user configuration. + * @param clusterConfig User provided cluster config, it overrides gear.conf when starting + * new applications. In most cases, you should not need to change it. If you do + * really need to change it, please use ClusterConfigSource(filePath) to + * construct the object, while filePath points to the .conf file. */ +case class AppDescription( + name: String, appMaster: String, userConfig: UserConfig, + clusterConfig: Config = ConfigFactory.empty()) -case class AppDescription(name : String, appMaster : String, userConfig: UserConfig, clusterConfig: Config = ConfigFactory.empty()) - +/** + * Each job, streaming or not streaming, need to provide an Application class. + * The master uses this class to start AppMaster. + */ trait Application { + + /** Name of this application, must be unique in the system */ def name: String + + /** Custom user configuration */ def userConfig(implicit system: ActorSystem): UserConfig + + /** + * AppMaster class, must have a constructor like this: + * this(appContext: AppMasterContext, app: AppDescription) + */ def appMaster: Class[_ <: ApplicationMaster] } object Application { - def apply[T <: ApplicationMaster](name: String, userConfig: UserConfig)(implicit tag: ClassTag[T]): Application = { - new DefaultApplication(name, userConfig, tag.runtimeClass.asInstanceOf[Class[_ <: ApplicationMaster]]) + def apply[T <: ApplicationMaster]( + name: String, userConfig: UserConfig)(implicit tag: ClassTag[T]): Application = { + new DefaultApplication(name, userConfig, + tag.runtimeClass.asInstanceOf[Class[_ <: ApplicationMaster]]) } - class DefaultApplication(override val name: String, inputUserConfig: UserConfig, val appMaster: Class[_ <: ApplicationMaster]) extends Application { + class DefaultApplication( + override val name: String, inputUserConfig: UserConfig, + val appMaster: Class[_ <: ApplicationMaster]) extends Application { override def userConfig(implicit system: ActorSystem): UserConfig = inputUserConfig } - def ApplicationToAppDescription(app: Application)(implicit system: ActorSystem): AppDescription = { + def ApplicationToAppDescription(app: Application)(implicit system: ActorSystem) + : AppDescription = { val filterJvmReservedKeys = ClusterConfig.filterOutDefaultConfig(system.settings.config) AppDescription(app.name, app.appMaster.getName, app.userConfig, filterJvmReservedKeys) } @@ -69,57 +89,57 @@ abstract class ApplicationMaster extends Actor /** * This contains context information when starting an AppMaster * - * @param appId: application instance id assigned, it is unique in the cluster - * @param username: The username who submitted this application - * @param resource: Resouce allocated to start this AppMaster daemon. AppMaster are allowed to - * request more resource from Master. - * @param appJar: application Jar. If the jar is already in classpath, then it can be None. - * @param masterProxy: The proxy to master actor, it will bridge the messages between appmaster and master - * @param registerData: The AppMaster are required to register this data back to Master by RegisterAppMaster - * + * @param appId application instance id assigned, it is unique in the cluster + * @param username The username who submitted this application + * @param resource Resouce allocated to start this AppMaster daemon. AppMaster are allowed to + * request more resource from Master. + * @param appJar application Jar. If the jar is already in classpath, then it can be None. + * @param masterProxy The proxy to master actor, it bridges the messages between appmaster + * and master + * @param registerData AppMaster are required to send this data to Master by when doing + * RegisterAppMaster. */ case class AppMasterContext( - appId : Int, - username : String, - resource : Resource, + appId: Int, + username: String, + resource: Resource, workerInfo: WorkerInfo, - appJar : Option[AppJar], - masterProxy : ActorRef, - registerData : AppMasterRegisterData) + appJar: Option[AppJar], + masterProxy: ActorRef, + registerData: AppMasterRegisterData) /** * Jar file container in the cluster * - * @param name: A meaningful name to represent this jar - * @param filePath: Where the jar file is stored. + * @param name A meaningful name to represent this jar + * @param filePath Where the jar file is stored. */ case class AppJar(name: String, filePath: FilePath) - /** - * TODO: ExecutorContext doesn't belong here. - * Need to move to other places + * Serves as the context to start an Executor JVM. */ -case class ExecutorContext(executorId : Int, worker: WorkerInfo, appId : Int, appName: String, - appMaster : ActorRef, resource : Resource) - +// TODO: ExecutorContext doesn't belong to this package in logic. +case class ExecutorContext( + executorId: Int, worker: WorkerInfo, appId: Int, appName: String, + appMaster: ActorRef, resource: Resource) /** - * TODO: ExecutorJVMConfig doesn't belong here. - * Need to move to other places - */ -/** - * @param classPath: When a worker create a executor, the parent worker's classpath will - * be automatically inherited, the application jar will also be added to runtime - * classpath automatically. Sometimes, you still want to add some extraclasspath, - * you can do this by specify classPath option. + * JVM configurations to start an Executor JVM. + * + * @param classPath When executor is created by a worker JVM, executor automatically inherits + * parent worker's classpath. Sometimes, you still want to add some extra + * classpath, you can do this by specify classPath option. * @param jvmArguments java arguments like -Dxx=yy * @param mainClass Executor main class name like io.gearpump.xx.AppMaster * @param arguments Executor command line arguments * @param jar application jar - * @param executorAkkaConfig Akka config used to initialize the actor system of this executor. It will - * use io.gearpump.util.Constants.GEARPUMP_CUSTOM_CONFIG_FILE to pass the config to executor - * process - * + * @param executorAkkaConfig Akka config used to initialize the actor system of this executor. + * It uses io.gearpump.util.Constants.GEARPUMP_CUSTOM_CONFIG_FILE + * to pass the config to executor process */ -case class ExecutorJVMConfig(classPath : Array[String], jvmArguments : Array[String], mainClass : String, arguments : Array[String], jar: Option[AppJar], username : String, executorAkkaConfig: Config = ConfigFactory.empty()) \ No newline at end of file +// TODO: ExecutorContext doesn't belong to this package in logic. +case class ExecutorJVMConfig( + classPath: Array[String], jvmArguments: Array[String], mainClass: String, + arguments: Array[String], jar: Option[AppJar], username: String, + executorAkkaConfig: Config = ConfigFactory.empty()) \ No newline at end of file diff --git a/core/src/main/scala/io/gearpump/cluster/ClusterConfig.scala b/core/src/main/scala/io/gearpump/cluster/ClusterConfig.scala index 64d5c4209..7bae6d699 100644 --- a/core/src/main/scala/io/gearpump/cluster/ClusterConfig.scala +++ b/core/src/main/scala/io/gearpump/cluster/ClusterConfig.scala @@ -19,24 +19,27 @@ package io.gearpump.cluster import java.io.File + import com.typesafe.config._ + import io.gearpump.util.Constants._ -import io.gearpump.util.{Util, FileUtils, Constants, LogUtil} -import scala.collection.JavaConversions._ +import io.gearpump.util.{Constants, FileUtils, LogUtil, Util} /** * * All Gearpump application should use this class to load configurations. * - * Compared with Akka built-in [[ConfigFactory]], this class will also - * resolve file gear.conf and geardefault.conf. + * Compared with Akka built-in com.typesafe.config.ConfigFactory, this class also + * resolve config from file gear.conf and geardefault.conf. * * Overriding order: + * {{{ * System Properties * > Custom configuration file (by using system property -Dgearpump.config.file) > * > gear.conf * > geardefault.conf * > reference.conf + * }}} */ object ClusterConfig { @@ -81,11 +84,10 @@ object ClusterConfig { load(configFile).ui } - /** * try to load system property gearpump.config.file, or use configFile */ - private def load(configFile: String) : Configs = { + private def load(configFile: String): Configs = { val file = Option(System.getProperty(GEARPUMP_CUSTOM_CONFIG_FILE)) file match { case Some(path) => @@ -100,7 +102,7 @@ object ClusterConfig { val APPLICATION = "application.conf" val LOG = LogUtil.getLogger(getClass) - def saveConfig(conf : Config, file : File) : Unit = { + def saveConfig(conf: Config, file: File): Unit = { val serialized = conf.root().render() FileUtils.write(file, serialized) } @@ -113,13 +115,13 @@ object ClusterConfig { } } - // filter JVM reserved keys and akka default reference.conf + /** filter JVM reserved keys and akka default reference.conf */ def filterOutDefaultConfig(input: Config): Config = { val updated = filterOutJvmReservedKeys(input) Util.filterOutOrigin(updated, "reference.conf") } - private[gearpump] def load(source: ClusterConfigSource) : Configs = { + private[gearpump] def load(source: ClusterConfigSource): Configs = { val systemProperties = getSystemProperties @@ -162,8 +164,8 @@ object ClusterConfig { ) private def getSystemProperties: Config = { - // exclude default java system properties - JVM_RESERVED_PROPERTIES.foldLeft(ConfigFactory.systemProperties()) {(config, property) => + // Excludes default java system properties + JVM_RESERVED_PROPERTIES.foldLeft(ConfigFactory.systemProperties()) { (config, property) => config.withoutPath(property) } } @@ -177,5 +179,6 @@ object ClusterConfig { filterJvmReservedKeys } - protected class Configs (val master: Config, val worker: Config, val ui: Config, val default: Config) + protected class Configs( + val master: Config, val worker: Config, val ui: Config, val default: Config) } \ No newline at end of file diff --git a/core/src/main/scala/io/gearpump/cluster/ClusterConfigSource.scala b/core/src/main/scala/io/gearpump/cluster/ClusterConfigSource.scala index 7e8a91d62..3a248d75f 100644 --- a/core/src/main/scala/io/gearpump/cluster/ClusterConfigSource.scala +++ b/core/src/main/scala/io/gearpump/cluster/ClusterConfigSource.scala @@ -19,9 +19,10 @@ package io.gearpump.cluster import java.io.File -import com.typesafe.config.{Config, ConfigFactory, ConfigParseOptions} import scala.language.implicitConversions +import com.typesafe.config.{Config, ConfigFactory, ConfigParseOptions} + /** * Data Source of ClusterConfig * diff --git a/core/src/main/scala/io/gearpump/cluster/ClusterMessage.scala b/core/src/main/scala/io/gearpump/cluster/ClusterMessage.scala index aac45ab49..5a42ea39d 100644 --- a/core/src/main/scala/io/gearpump/cluster/ClusterMessage.scala +++ b/core/src/main/scala/io/gearpump/cluster/ClusterMessage.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,26 +18,25 @@ package io.gearpump.cluster +import scala.util.Try + import akka.actor.ActorRef import com.typesafe.config.Config -import io.gearpump.{WorkerId, TimeStamp} + +import io.gearpump.TimeStamp import io.gearpump.cluster.MasterToAppMaster.AppMasterStatus import io.gearpump.cluster.master.MasterSummary import io.gearpump.cluster.scheduler.{Resource, ResourceAllocation, ResourceRequest} -import io.gearpump.cluster.worker.WorkerSummary +import io.gearpump.cluster.worker.{WorkerId, WorkerSummary} import io.gearpump.metrics.Metrics.MetricType -import scala.util.Try - -/** - * Application Flow - */ - object ClientToMaster { case object AddMaster case class AddWorker(count: Int) case class RemoveMaster(masterContainerId: String) case class RemoveWorker(workerContainerId: String) + + /** Command result of AddMaster, RemoveMaster, and etc... */ case class CommandResult(success: Boolean, exception: String = null) { override def toString: String = { val tag = getClass.getSimpleName @@ -48,50 +47,89 @@ object ClientToMaster { } } } - case class SubmitApplication(appDescription: AppDescription, appJar: Option[AppJar], username : String = System.getProperty("user.name")) + + /** Submit an application to master */ + case class SubmitApplication( + appDescription: AppDescription, appJar: Option[AppJar], + username: String = System.getProperty("user.name")) + case class RestartApplication(appId: Int) case class ShutdownApplication(appId: Int) + + /** Client send ResolveAppId to Master to resolves AppMaster actor path by providing appId */ case class ResolveAppId(appId: Int) + /** Client send ResolveWorkerId to master to get the Actor path of worker. */ case class ResolveWorkerId(workerId: WorkerId) + /** Get an active Jar store to upload job jars, like wordcount.jar */ case object GetJarStoreServer + /** Service address of JarStore */ case class JarStoreServerAddress(url: String) + /** Query AppMaster config by providing appId */ case class QueryAppMasterConfig(appId: Int) + /** Query worker config */ case class QueryWorkerConfig(workerId: WorkerId) + /** Query master config */ case object QueryMasterConfig + /** Options for read the metrics from the cluster */ object ReadOption { type ReadOption = String val Key: String = "readOption" + /** Read the latest record of the metrics, only return 1 record for one metric name (id) */ val ReadLatest: ReadOption = "readLatest" + /** Read recent metrics from cluster, typically it contains metrics in 5 minutes */ val ReadRecent = "readRecent" + /** + * Read the history metrics, typically it contains metrics for 48 hours + * + * NOTE: Each hour only contain one or two data points. + */ val ReadHistory = "readHistory" } + /** Query history metrics from master or app master. */ + case class QueryHistoryMetrics( + path: String, readOption: ReadOption.ReadOption = ReadOption.ReadLatest, + aggregatorClazz: String = "", options: Map[String, String] = Map.empty[String, String]) - case class QueryHistoryMetrics(path: String, readOption: ReadOption.ReadOption = ReadOption.ReadLatest, aggregatorClazz: String = "", options: Map[String, String] = Map.empty[String, String]) - + /** + * If there are message loss, the clock would pause for a while. This message is used to + * pin-point which task has stalling clock value, and usually it means something wrong on + * that machine. + */ case class GetStallingTasks(appId: Int) + /** + * Request app master for a short list of cluster app that administrators should be aware of. + */ case class GetLastFailure(appId: Int) } object MasterToClient { - case class SubmitApplicationResult(appId : Try[Int]) + + /** Result of SubmitApplication */ + // TODO: Merge with SubmitApplicationResultValue and change this to (appId: Option, ex: Exception) + case class SubmitApplicationResult(appId: Try[Int]) + case class SubmitApplicationResultValue(appId: Int) - case class ShutdownApplicationResult(appId : Try[Int]) + + case class ShutdownApplicationResult(appId: Try[Int]) case class ReplayApplicationResult(appId: Try[Int]) + + /** Return Actor ref of app master */ case class ResolveAppIdResult(appMaster: Try[ActorRef]) + /** Return Actor ref of worker */ case class ResolveWorkerIdResult(worker: Try[ActorRef]) case class AppMasterConfig(config: Config) @@ -102,25 +140,64 @@ object MasterToClient { case class HistoryMetricsItem(time: TimeStamp, value: MetricType) + /** + * History metrics returned from master, worker, or app master. + * + * All metric items are organized like a tree, path is used to navigate through the tree. + * For example, when querying with path == "executor0.task1.throughput*", the metrics + * provider picks metrics whose source matches the path. + * + * @param path The path client provided. The returned metrics are the result query of this path. + * @param metrics The detailed metrics. + */ case class HistoryMetrics(path: String, metrics: List[HistoryMetricsItem]) + /** Return the last error of this streaming application job */ case class LastFailure(time: TimeStamp, error: String) } trait AppMasterRegisterData object AppMasterToMaster { - case class RegisterAppMaster(appMaster: ActorRef, registerData : AppMasterRegisterData) + + /** + * Register an AppMaster by providing a ActorRef, and registerData + * @param registerData The registerData is provided by Master when starting the app master. + * App master should return the registerData back to master. + * Typically registerData hold some context information for this app Master. + */ + + case class RegisterAppMaster(appMaster: ActorRef, registerData: AppMasterRegisterData) + case class InvalidAppMaster(appId: Int, appMaster: String, reason: Throwable) + case class RequestResource(appId: Int, request: ResourceRequest) + /** + * Each application job can save some data in the distributed cluster storage on master nodes. + * + * @param appId App Id of the client application who send the request. + * @param key Key name + * @param value Value to store on distributed cluster storage on master nodes + */ case class SaveAppData(appId: Int, key: String, value: Any) + + /** The application specific data is successfully stored */ case object AppDataSaved + + /** Fail to store the application data */ case object SaveAppDataFailed + /** Fetch the application specific data that stored previously */ case class GetAppData(appId: Int, key: String) + + /** The KV data returned for query GetAppData */ case class GetAppDataResult(key: String, value: Any) + /** + * AppMasterSummary returned to REST API query. Streaming and Non-streaming + * have very different application info. AppMasterSummary is the common interface. + */ trait AppMasterSummary { def appType: String def appId: Int @@ -132,37 +209,43 @@ object AppMasterToMaster { def user: String } + /** Represents a generic application that is not a streaming job */ case class GeneralAppMasterSummary( - appId: Int, - appType: String = "general", - appName: String = null, - actorPath: String = null, - status: AppMasterStatus = MasterToAppMaster.AppMasterActive, - startTime: TimeStamp = 0L, - uptime: TimeStamp = 0L, - user: String = null) + appId: Int, + appType: String = "general", + appName: String = null, + actorPath: String = null, + status: AppMasterStatus = MasterToAppMaster.AppMasterActive, + startTime: TimeStamp = 0L, + uptime: TimeStamp = 0L, + user: String = null) extends AppMasterSummary + /** Fetches the list of workers from Master */ case object GetAllWorkers + + /** Get worker data of workerId */ case class GetWorkerData(workerId: WorkerId) + + /** Response to GetWorkerData */ case class WorkerData(workerDescription: WorkerSummary) + /** Get Master data */ case object GetMasterData + + /** Response to GetMasterData */ case class MasterData(masterDescription: MasterSummary) } object MasterToAppMaster { - case class ResourceAllocated(allocations: Array[ResourceAllocation]){ - override def equals(other: Any): Boolean = { - other match { - case that: ResourceAllocated => - allocations.sortBy(_.workerId).sameElements(that.allocations.sortBy(_.workerId)) - case _ => - false - } - } - } + + /** Resource allocated for application xx */ + case class ResourceAllocated(allocations: Array[ResourceAllocation]) + + /** Master confirm reception of RegisterAppMaster message */ case class AppMasterRegistered(appId: Int) + + /** Shutdown the application job */ case object ShutdownAppMaster type AppMasterStatus = String @@ -171,7 +254,11 @@ object MasterToAppMaster { val AppMasterNonExist: AppMasterStatus = "nonexist" sealed trait StreamingType - case class AppMasterData(status: AppMasterStatus, appId: Int = 0, appName: String = null, appMasterPath: String = null, workerPath: String = null, submissionTime: TimeStamp = 0, startTime: TimeStamp = 0, finishTime: TimeStamp = 0, user: String = null) + case class AppMasterData( + status: AppMasterStatus, appId: Int = 0, appName: String = null, appMasterPath: String = null, + workerPath: String = null, submissionTime: TimeStamp = 0, startTime: TimeStamp = 0, + finishTime: TimeStamp = 0, user: String = null) + case class AppMasterDataRequest(appId: Int, detail: Boolean = false) case class AppMastersData(appMasters: List[AppMasterData]) @@ -185,8 +272,10 @@ object MasterToAppMaster { } object AppMasterToWorker { - case class LaunchExecutor(appId: Int, executorId: Int, resource: Resource, executorJvmConfig: ExecutorJVMConfig) - case class ShutdownExecutor(appId : Int, executorId : Int, reason : String) + case class LaunchExecutor( + appId: Int, executorId: Int, resource: Resource, executorJvmConfig: ExecutorJVMConfig) + + case class ShutdownExecutor(appId: Int, executorId: Int, reason: String) case class ChangeExecutorResource(appId: Int, executorId: Int, resource: Resource) } @@ -196,4 +285,3 @@ object WorkerToAppMaster { case class ShutdownExecutorFailed(reason: String = null, ex: Throwable = null) } - diff --git a/core/src/main/scala/io/gearpump/cluster/UserConfig.scala b/core/src/main/scala/io/gearpump/cluster/UserConfig.scala index 0756d5496..61de1dd97 100644 --- a/core/src/main/scala/io/gearpump/cluster/UserConfig.scala +++ b/core/src/main/scala/io/gearpump/cluster/UserConfig.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,14 +18,15 @@ package io.gearpump.cluster -import akka.actor.{ExtendedActorSystem, ActorSystem} +import akka.actor.{ActorSystem, ExtendedActorSystem} import akka.serialization.JavaSerializer + import io.gearpump.google.common.io.BaseEncoding /** * Immutable configuration */ -final class UserConfig(private val _config: Map[String, String]) extends Serializable{ +final class UserConfig(private val _config: Map[String, String]) extends Serializable { def withBoolean(key: String, value: Boolean): UserConfig = { new UserConfig(_config + (key -> value.toString)) @@ -39,7 +40,7 @@ final class UserConfig(private val _config: Map[String, String]) extends Serial new UserConfig(_config + (key -> value.toString)) } - def withInt(key: String, value: Int) : UserConfig = { + def withInt(key: String, value: Int): UserConfig = { new UserConfig(_config + (key -> value.toString)) } @@ -77,7 +78,7 @@ final class UserConfig(private val _config: Map[String, String]) extends Serial _config.get(key).map(_.toFloat) } - def getInt(key : String) : Option[Int] = { + def getInt(key: String): Option[Int] = { _config.get(key).map(_.toInt) } @@ -85,11 +86,11 @@ final class UserConfig(private val _config: Map[String, String]) extends Serial _config.get(key).map(_.toLong) } - def getString(key : String) : Option[String] = { + def getString(key: String): Option[String] = { _config.get(key) } - def getBytes(key: String) : Option[Array[Byte]] = { + def getBytes(key: String): Option[Array[Byte]] = { _config.get(key).map(BaseEncoding.base64().decode(_)) } @@ -101,31 +102,35 @@ final class UserConfig(private val _config: Map[String, String]) extends Serial } } + // scalastyle:off line.size.limit /** - * This will de-serialize value to object instance + * This de-serializes value to object instance * * To do de-serialization, this requires an implicit ActorSystem, as * the ActorRef and possibly other akka classes deserialization * requires an implicit ActorSystem. * - * @see [[http://doc.akka.io/docs/akka/snapshot/scala/serialization.html#A_Word_About_Java_Serialization]] + * See Link: + * http://doc.akka.io/docs/akka/snapshot/scala/serialization.html#A_Word_About_Java_Serialization */ - def getValue[T](key: String)(implicit system: ActorSystem): Option[T] = { + + def getValue[T](key: String)(implicit system: ActorSystem): Option[T] = { val serializer = new JavaSerializer(system.asInstanceOf[ExtendedActorSystem]) _config.get(key).map(BaseEncoding.base64().decode(_)) .map(serializer.fromBinary(_).asInstanceOf[T]) } /** - * This will serialize the object and store it as string. + * This serializes the object and store it as string. * * To do serialization, this requires an implicit ActorSystem, as * the ActorRef and possibly other akka classes serialization * requires an implicit ActorSystem. * - * @see [[http://doc.akka.io/docs/akka/snapshot/scala/serialization.html#A_Word_About_Java_Serialization]] + * See Link: + * http://doc.akka.io/docs/akka/snapshot/scala/serialization.html#A_Word_About_Java_Serialization */ - def withValue[T<: AnyRef](key: String, value: T)(implicit system: ActorSystem): UserConfig = { + def withValue[T <: AnyRef](key: String, value: T)(implicit system: ActorSystem): UserConfig = { if (null == value) { this @@ -136,8 +141,9 @@ final class UserConfig(private val _config: Map[String, String]) extends Serial this.withString(key, encoded) } } + // scalastyle:on line.size.limit - def withConfig(other: UserConfig) = { + def withConfig(other: UserConfig): UserConfig = { if (null == other) { this } else { @@ -146,11 +152,11 @@ final class UserConfig(private val _config: Map[String, String]) extends Serial } } -object UserConfig{ +object UserConfig { - def empty = new UserConfig(Map.empty[String, String]) + def empty: UserConfig = new UserConfig(Map.empty[String, String]) - def apply(config : Map[String, String]) = new UserConfig(config) + def apply(config: Map[String, String]): UserConfig = new UserConfig(config) def unapply(config: UserConfig): Option[Map[String, String]] = Option(config._config) } diff --git a/core/src/main/scala/io/gearpump/cluster/appmaster/AppMasterRuntimeEnvironment.scala b/core/src/main/scala/io/gearpump/cluster/appmaster/AppMasterRuntimeEnvironment.scala index 1e3467829..4e8582be6 100644 --- a/core/src/main/scala/io/gearpump/cluster/appmaster/AppMasterRuntimeEnvironment.scala +++ b/core/src/main/scala/io/gearpump/cluster/appmaster/AppMasterRuntimeEnvironment.scala @@ -19,32 +19,34 @@ package io.gearpump.cluster.appmaster import akka.actor._ + import io.gearpump.cluster.AppMasterToMaster.RegisterAppMaster import io.gearpump.cluster.appmaster.AppMasterRuntimeEnvironment._ import io.gearpump.cluster.appmaster.ExecutorSystemScheduler.{Session, StartExecutorSystems} import io.gearpump.cluster.appmaster.MasterConnectionKeeper.MasterConnectionStatus._ import io.gearpump.cluster.master.MasterProxy -import io.gearpump.cluster.{AppMasterContext, AppDescription} +import io.gearpump.cluster.{AppDescription, AppMasterContext} import io.gearpump.util.LogUtil /** * This serves as runtime environment for AppMaster. - * When starting an AppMaster, we need to setup the connection to master, and prepare other environemnts. + * When starting an AppMaster, we need to setup the connection to master, + * and prepare other environments. * * This also extend the function of Master, by providing a scheduler service for Executor System. * AppMaster can ask Master for executor system directly. details like requesting resource, - * contacting worker to start a process, and then starting an executor system is hidden from AppMaster. + * contacting worker to start a process, and then starting an executor system is hidden from + * AppMaster. * * Please use AppMasterRuntimeEnvironment.props() to construct this actor. - * */ private[appmaster] -class AppMasterRuntimeEnvironment ( +class AppMasterRuntimeEnvironment( appContextInput: AppMasterContext, app: AppDescription, masters: Iterable[ActorPath], masterFactory: (AppId, MasterActorRef) => Props, - appMasterFactory: (AppMasterContext, AppDescription)=> Props, + appMasterFactory: (AppMasterContext, AppDescription) => Props, masterConnectionKeeperFactory: (MasterActorRef, RegisterAppMaster, ListenerActorRef) => Props) extends Actor { @@ -52,15 +54,18 @@ class AppMasterRuntimeEnvironment ( private val LOG = LogUtil.getLogger(getClass, app = appId) import scala.concurrent.duration._ - private val master = context.actorOf(masterFactory(appId, context.actorOf(Props(new MasterProxy(masters, 30 seconds))))) + + private val master = context.actorOf( + masterFactory(appId, context.actorOf(Props(new MasterProxy(masters, 30.seconds))))) private val appContext = appContextInput.copy(masterProxy = master) - //create appMaster proxy to receive command and forward to appmaster + // Create appMaster proxy to receive command and forward to appmaster private val appMaster = context.actorOf(appMasterFactory(appContext, app)) context.watch(appMaster) private val registerAppMaster = RegisterAppMaster(appMaster, appContext.registerData) - private val masterConnectionKeeper = context.actorOf(masterConnectionKeeperFactory(master, registerAppMaster, self)) + private val masterConnectionKeeper = context.actorOf( + masterConnectionKeeperFactory(master, registerAppMaster, self)) context.watch(masterConnectionKeeper) def receive: Receive = { @@ -72,20 +77,21 @@ class AppMasterRuntimeEnvironment ( context.stop(self) case Terminated(actor) => actor match { case `appMaster` => - LOG.error (s"AppMaster ${appId} is stopped, shutdown myself") - context.stop (self) + LOG.error(s"AppMaster ${appId} is stopped, shutdown myself") + context.stop(self) case `masterConnectionKeeper` => - LOG.error (s"Master connection keeper is stopped, appId: ${appId}, shutdown myself") - context.stop (self) - case _ => //skip + LOG.error(s"Master connection keeper is stopped, appId: ${appId}, shutdown myself") + context.stop(self) + case _ => // Skip } } } object AppMasterRuntimeEnvironment { - - def props(masters: Iterable[ActorPath], app : AppDescription, appContextInput: AppMasterContext): Props = { + def props( + masters: Iterable[ActorPath], app: AppDescription, appContextInput: AppMasterContext) + : Props = { val master = (appId: AppId, masterProxy: MasterActorRef) => MasterWithExecutorSystemProvider.props(appId, masterProxy) @@ -93,26 +99,25 @@ object AppMasterRuntimeEnvironment { val appMaster = (appContext: AppMasterContext, app: AppDescription) => LazyStartAppMaster.props(appContext, app) - val masterConnectionKeeper = - (master: MasterActorRef, registerAppMaster: RegisterAppMaster, listener: ListenerActorRef) => - Props(new MasterConnectionKeeper(registerAppMaster, master, masterStatusListener = listener)) + val masterConnectionKeeper = (master: MasterActorRef, registerAppMaster: + RegisterAppMaster, listener: ListenerActorRef) => Props(new MasterConnectionKeeper( + registerAppMaster, master, masterStatusListener = listener)) Props(new AppMasterRuntimeEnvironment( appContextInput, app, masters, master, appMaster, masterConnectionKeeper)) } /** - * This behavior as AppMaster, and will lazy start the real AppMaster. When real AppMaster - * is not started yet, all messages will be stashed. The stashed messages will be forwarded to - * real AppMaster when it is started. + * This behavior like a AppMaster. Under the hood, It start start the real AppMaster in a lazy + * way. When real AppMaster is not started yet, all messages are stashed. The stashed + * messages are forwarded to real AppMaster when the real AppMaster is started. * * Please use LazyStartAppMaster.props to construct this actor * - * @param appId * @param appMasterProps underlying AppMaster Props */ private[appmaster] - class LazyStartAppMaster (appId: Int, appMasterProps: Props) extends Actor with Stash { + class LazyStartAppMaster(appId: Int, appMasterProps: Props) extends Actor with Stash { private val LOG = LogUtil.getLogger(getClass, app = appId) @@ -151,14 +156,11 @@ object AppMasterRuntimeEnvironment { private[appmaster] case object StartAppMaster - /** * This enhance Master by providing new service: StartExecutorSystems * - * * Please use MasterWithExecutorSystemProvider.props to construct this actor + * Please use MasterWithExecutorSystemProvider.props to construct this actor * - * @param master - * @param executorSystemProviderProps */ private[appmaster] class MasterWithExecutorSystemProvider(master: ActorRef, executorSystemProviderProps: Props) @@ -168,7 +170,7 @@ object AppMasterRuntimeEnvironment { override def receive: Receive = { case request: StartExecutorSystems => - executorSystemProvider forward request + executorSystemProvider forward request case msg => master forward msg } @@ -181,13 +183,12 @@ object AppMasterRuntimeEnvironment { val executorSystemLauncher = (appId: Int, session: Session) => Props(new ExecutorSystemLauncher(appId, session)) - val scheduler = Props(new ExecutorSystemScheduler(appId, master, executorSystemLauncher)) + val scheduler = Props(new ExecutorSystemScheduler(appId, master, executorSystemLauncher)) Props(new MasterWithExecutorSystemProvider(master, scheduler)) } } - private[appmaster] type AppId = Int private[appmaster] type MasterActorRef = ActorRef private[appmaster] type ListenerActorRef = ActorRef diff --git a/core/src/main/scala/io/gearpump/cluster/appmaster/AppMasterRuntimeInfo.scala b/core/src/main/scala/io/gearpump/cluster/appmaster/AppMasterRuntimeInfo.scala index 11414cf1c..5b3e0c59b 100644 --- a/core/src/main/scala/io/gearpump/cluster/appmaster/AppMasterRuntimeInfo.scala +++ b/core/src/main/scala/io/gearpump/cluster/appmaster/AppMasterRuntimeInfo.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,21 +15,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package io.gearpump.cluster.appmaster import akka.actor.ActorRef import com.typesafe.config.Config + import io.gearpump._ import io.gearpump.cluster.AppMasterRegisterData +/** Run time info used to start an AppMaster */ case class AppMasterRuntimeInfo( - appId: Int, - // appName is the unique Id for an application - appName: String, - worker : ActorRef = null, - user: String = null, - submissionTime: TimeStamp = 0, - startTime: TimeStamp = 0, - finishTime: TimeStamp = 0, - config: Config = null) + appId: Int, + // AppName is the unique Id for an application + appName: String, + worker: ActorRef = null, + user: String = null, + submissionTime: TimeStamp = 0, + startTime: TimeStamp = 0, + finishTime: TimeStamp = 0, + config: Config = null) extends AppMasterRegisterData diff --git a/core/src/main/scala/io/gearpump/cluster/appmaster/ApplicationState.scala b/core/src/main/scala/io/gearpump/cluster/appmaster/ApplicationState.scala index 430c9580e..3b967f4df 100644 --- a/core/src/main/scala/io/gearpump/cluster/appmaster/ApplicationState.scala +++ b/core/src/main/scala/io/gearpump/cluster/appmaster/ApplicationState.scala @@ -18,28 +18,30 @@ package io.gearpump.cluster.appmaster -import io.gearpump.cluster.{AppJar, AppDescription} +import io.gearpump.cluster.{AppDescription, AppJar} /** - * This state will be persisted across the masters. - */ -case class ApplicationState(appId : Int, appName: String, attemptId : Int, app : AppDescription, jar: Option[AppJar], username : String, state : Any) extends Serializable { + * This state for single application, it is be distributed across the masters. + */ +case class ApplicationState( + appId: Int, appName: String, attemptId: Int, app: AppDescription, jar: Option[AppJar], + username: String, state: Any) extends Serializable { - override def equals(other: Any): Boolean = { - other match { - case that: ApplicationState => - if (appId == that.appId && attemptId == that.attemptId) { - true - } else { - false - } - case _ => - false - } - } + override def equals(other: Any): Boolean = { + other match { + case that: ApplicationState => + if (appId == that.appId && attemptId == that.attemptId) { + true + } else { + false + } + case _ => + false + } + } - override def hashCode: Int = { - import akka.routing.MurmurHash._ - extendHash(appId, attemptId, startMagicA, startMagicB) - } - } + override def hashCode: Int = { + import akka.routing.MurmurHash._ + extendHash(appId, attemptId, startMagicA, startMagicB) + } +} diff --git a/core/src/main/scala/io/gearpump/cluster/appmaster/ExecutorSystem.scala b/core/src/main/scala/io/gearpump/cluster/appmaster/ExecutorSystem.scala index 400b61cc4..6fcb5e718 100644 --- a/core/src/main/scala/io/gearpump/cluster/appmaster/ExecutorSystem.scala +++ b/core/src/main/scala/io/gearpump/cluster/appmaster/ExecutorSystem.scala @@ -19,22 +19,25 @@ package io.gearpump.cluster.appmaster import akka.actor.{ActorRef, Address, PoisonPill} -import io.gearpump.WorkerId + import io.gearpump.cluster.scheduler.Resource +import io.gearpump.cluster.worker.WorkerId import io.gearpump.util.ActorSystemBooter.BindLifeCycle case class WorkerInfo(workerId: WorkerId, ref: ActorRef) /** - * This contains JVM configurations to start an executor system + * Configurations to start an executor system on remote machine + * + * @param address Remote address where we start an Actor System. */ case class ExecutorSystem(executorSystemId: Int, address: Address, daemon: -ActorRef, resource: Resource, worker: WorkerInfo) { + ActorRef, resource: Resource, worker: WorkerInfo) { def bindLifeCycleWith(actor: ActorRef): Unit = { daemon ! BindLifeCycle(actor) } - def shutdown: Unit = { + def shutdown(): Unit = { daemon ! PoisonPill } } \ No newline at end of file diff --git a/core/src/main/scala/io/gearpump/cluster/appmaster/ExecutorSystemLauncher.scala b/core/src/main/scala/io/gearpump/cluster/appmaster/ExecutorSystemLauncher.scala index 80af74832..78432f485 100644 --- a/core/src/main/scala/io/gearpump/cluster/appmaster/ExecutorSystemLauncher.scala +++ b/core/src/main/scala/io/gearpump/cluster/appmaster/ExecutorSystemLauncher.scala @@ -18,41 +18,42 @@ package io.gearpump.cluster.appmaster +import scala.concurrent.duration._ + import akka.actor._ +import org.slf4j.Logger + import io.gearpump.cluster.AppMasterToWorker.LaunchExecutor import io.gearpump.cluster.ExecutorJVMConfig -import io.gearpump.cluster.scheduler.Resource -import io.gearpump.util.{Constants, ActorSystemBooter, ActorUtil, LogUtil} import io.gearpump.cluster.WorkerToAppMaster._ import io.gearpump.cluster.appmaster.ExecutorSystemLauncher._ import io.gearpump.cluster.appmaster.ExecutorSystemScheduler.{ExecutorSystemJvmConfig, Session} -import io.gearpump.util.ActorSystemBooter.{ActorSystemRegistered, BindLifeCycle, RegisterActorSystem} -import org.slf4j.Logger - -import scala.concurrent.duration._ +import io.gearpump.cluster.scheduler.Resource +import io.gearpump.util.ActorSystemBooter.{ActorSystemRegistered, RegisterActorSystem} +import io.gearpump.util.{ActorSystemBooter, ActorUtil, Constants, LogUtil} /** * This launches single executor system on target worker. * * Please use ExecutorSystemLauncher.props() to construct this actor * - * @param appId * @param session The session that request to launch executor system */ private[appmaster] -class ExecutorSystemLauncher (appId: Int, session: Session) extends Actor { +class ExecutorSystemLauncher(appId: Int, session: Session) extends Actor { private val LOG: Logger = LogUtil.getLogger(getClass) val scheduler = context.system.scheduler implicit val executionContext = context.dispatcher - val timeoutSetting = context.system.settings.config.getInt(Constants.GEARPUMP_START_EXECUTOR_SYSTEM_TIMEOUT_MS) + private val systemConfig = context.system.settings.config + val timeoutSetting = systemConfig.getInt(Constants.GEARPUMP_START_EXECUTOR_SYSTEM_TIMEOUT_MS) - val timeout = scheduler.scheduleOnce(timeoutSetting milliseconds, + val timeout = scheduler.scheduleOnce(timeoutSetting.milliseconds, self, LaunchExecutorSystemTimeout(session)) - def receive : Receive = waitForLaunchCommand + def receive: Receive = waitForLaunchCommand def waitForLaunchCommand: Receive = { case LaunchExecutorSystem(worker, executorSystemId, resource) => @@ -61,23 +62,27 @@ class ExecutorSystemLauncher (appId: Int, session: Session) extends Actor { .map(getExecutorJvmConfig(_, s"app${appId}system${executorSystemId}", launcherPath)).orNull val launch = LaunchExecutor(appId, executorSystemId, resource, jvmConfig) - LOG.info(s"Launching Executor ...appId: $appId, executorSystemId: $executorSystemId, slots: ${resource.slots} on worker $worker") + LOG.info(s"Launching Executor ...appId: $appId, executorSystemId: $executorSystemId, " + + s"slots: ${resource.slots} on worker $worker") worker.ref ! launch context.become(waitForActorSystemToStart(sender, launch, worker, executorSystemId)) } - def waitForActorSystemToStart(replyTo: ActorRef, launch: LaunchExecutor, worker: WorkerInfo, executorSystemId: Int) : Receive = { + def waitForActorSystemToStart( + replyTo: ActorRef, launch: LaunchExecutor, worker: WorkerInfo, executorSystemId: Int) + : Receive = { case RegisterActorSystem(systemPath) => import launch._ timeout.cancel() LOG.info(s"Received RegisterActorSystem $systemPath for session ${session.requestor}") sender ! ActorSystemRegistered(worker.ref) - val system = ExecutorSystem(executorId, AddressFromURIString(systemPath), sender, resource, worker) + val system = + ExecutorSystem(executorId, AddressFromURIString(systemPath), sender, resource, worker) replyTo ! LaunchExecutorSystemSuccess(system, session) context.stop(self) - case reject @ ExecutorLaunchRejected(reason, ex) => - LOG.error(s"Executor Launch ${launch.resource} failed reason：$reason", ex) + case reject@ExecutorLaunchRejected(reason, ex) => + LOG.error(s"Executor Launch ${launch.resource} failed reason: $reason", ex) replyTo ! LaunchExecutorSystemRejected(launch.resource, reason, session) context.stop(self) case timeout: LaunchExecutorSystemTimeout => @@ -89,6 +94,7 @@ class ExecutorSystemLauncher (appId: Int, session: Session) extends Actor { private[appmaster] object ExecutorSystemLauncher { + case class LaunchExecutorSystem(worker: WorkerInfo, systemId: Int, resource: Resource) case class LaunchExecutorSystemSuccess(system: ExecutorSystem, session: Session) diff --git a/core/src/main/scala/io/gearpump/cluster/appmaster/ExecutorSystemScheduler.scala b/core/src/main/scala/io/gearpump/cluster/appmaster/ExecutorSystemScheduler.scala index aa980b5b9..c5ec6007f 100644 --- a/core/src/main/scala/io/gearpump/cluster/appmaster/ExecutorSystemScheduler.scala +++ b/core/src/main/scala/io/gearpump/cluster/appmaster/ExecutorSystemScheduler.scala @@ -18,29 +18,29 @@ package io.gearpump.cluster.appmaster +import scala.concurrent.duration._ + import akka.actor._ import com.typesafe.config.Config -import io.gearpump.WorkerId + import io.gearpump.cluster.AppMasterToMaster.RequestResource import io.gearpump.cluster.MasterToAppMaster.ResourceAllocated import io.gearpump.cluster._ import io.gearpump.cluster.appmaster.ExecutorSystemLauncher._ import io.gearpump.cluster.appmaster.ExecutorSystemScheduler._ import io.gearpump.cluster.scheduler.{ResourceAllocation, ResourceRequest} +import io.gearpump.cluster.worker.WorkerId import io.gearpump.util.{Constants, LogUtil} -import scala.concurrent.duration._ - /** * ExecutorSystem is also a type of resource, this class schedules ExecutorSystem for AppMaster. * AppMaster can use this class to directly request a live executor actor systems. The communication * in the background with Master and Worker is hidden from AppMaster. * * Please use ExecutorSystemScheduler.props() to construct this actor - * -*/ + */ private[appmaster] -class ExecutorSystemScheduler (appId: Int, masterProxy: ActorRef, +class ExecutorSystemScheduler(appId: Int, masterProxy: ActorRef, executorSystemLauncher: (Int, Session) => Props) extends Actor { private val LOG = LogUtil.getLogger(getClass, app = appId) @@ -50,18 +50,22 @@ class ExecutorSystemScheduler (appId: Int, masterProxy: ActorRef, var resourceAgents = Map.empty[Session, ActorRef] - def receive: Receive = clientCommands orElse resourceAllocationMessageHandler orElse executorSystemMessageHandler + def receive: Receive = { + clientCommands orElse resourceAllocationMessageHandler orElse executorSystemMessageHandler + } def clientCommands: Receive = { case start: StartExecutorSystems => - LOG.info(s"starting executor systems (ExecutorSystemConfig(${start.executorSystemConfig}), Resources(${start.resources.mkString(",")}))") + LOG.info(s"starting executor systems (ExecutorSystemConfig(${start.executorSystemConfig}), " + + s"Resources(${start.resources.mkString(",")}))") val requestor = sender() val executorSystemConfig = start.executorSystemConfig - val session = Session(requestor, executorSystemConfig) - val agent = resourceAgents.getOrElse(session, context.actorOf(Props(new ResourceAgent(masterProxy, session)))) + val session = Session(requestor, executorSystemConfig) + val agent = resourceAgents.getOrElse(session, + context.actorOf(Props(new ResourceAgent(masterProxy, session)))) resourceAgents = resourceAgents + (session -> agent) - start.resources.foreach {resource => + start.resources.foreach { resource => agent ! RequestResource(appId, resource) } @@ -87,14 +91,15 @@ class ExecutorSystemScheduler (appId: Int, masterProxy: ActorRef, } } - def executorSystemMessageHandler : Receive = { + def executorSystemMessageHandler: Receive = { case LaunchExecutorSystemSuccess(system, session) => if (isSessionAlive(session)) { LOG.info("LaunchExecutorSystemSuccess, send back to " + session.requestor) system.bindLifeCycleWith(self) session.requestor ! ExecutorSystemStarted(system, session.executorSystemJvmConfig.jar) } else { - LOG.error("We get a ExecutorSystem back, but resource requestor is no longer valid. Will shutdown the allocated system") + LOG.error("We get a ExecutorSystem back, but resource requestor is no longer valid. " + + "Will shutdown the allocated system") system.shutdown } case LaunchExecutorSystemTimeout(session) => @@ -105,8 +110,11 @@ class ExecutorSystemScheduler (appId: Int, masterProxy: ActorRef, case LaunchExecutorSystemRejected(resource, reason, session) => if (isSessionAlive(session)) { - LOG.error(s"Failed to launch executor system, due to $reason, will ask master to allocate new resources $resource") - resourceAgents.get(session).map(_ ! RequestResource(appId, ResourceRequest(resource, WorkerId.unspecified))) + LOG.error(s"Failed to launch executor system, due to $reason, " + + s"will ask master to allocate new resources $resource") + resourceAgents.get(session).map { resourceAgent: ActorRef => + resourceAgent ! RequestResource(appId, ResourceRequest(resource, WorkerId.unspecified)) + } } } @@ -117,27 +125,28 @@ class ExecutorSystemScheduler (appId: Int, masterProxy: ActorRef, object ExecutorSystemScheduler { - case class StartExecutorSystems(resources: Array[ResourceRequest], executorSystemConfig: ExecutorSystemJvmConfig) + case class StartExecutorSystems( + resources: Array[ResourceRequest], executorSystemConfig: ExecutorSystemJvmConfig) + case class ExecutorSystemStarted(system: ExecutorSystem, boundedJar: Option[AppJar]) + case class StopExecutorSystem(system: ExecutorSystem) - case object StartExecutorSystemTimeout - case class ExecutorSystemJvmConfig(classPath : Array[String], jvmArguments : Array[String], - jar: Option[AppJar], username : String, executorAkkaConfig: Config = null) + case object StartExecutorSystemTimeout + case class ExecutorSystemJvmConfig(classPath: Array[String], jvmArguments: Array[String], + jar: Option[AppJar], username: String, executorAkkaConfig: Config = null) /** * For each client which ask for an executor system, the scheduler will create a session for it. * - * @param requestor - * @param executorSystemJvmConfig */ - private [appmaster] + private[appmaster] case class Session(requestor: ActorRef, executorSystemJvmConfig: ExecutorSystemJvmConfig) /** * This is a agent for session to request resource - * @param master + * * @param session the original requester of the resource requests */ private[appmaster] @@ -145,17 +154,19 @@ object ExecutorSystemScheduler { private var resourceRequestor: ActorRef = null var timeOutClock: Cancellable = null private var unallocatedResource: Int = 0 + import context.dispatcher - import Constants._ - val timeout = context.system.settings.config.getInt(GEARPUMP_RESOURCE_ALLOCATION_TIMEOUT) + import io.gearpump.util.Constants._ + val timeout = context.system.settings.config.getInt(GEARPUMP_RESOURCE_ALLOCATION_TIMEOUT) def receive: Receive = { case request: RequestResource => unallocatedResource += request.request.resource.slots Option(timeOutClock).map(_.cancel) - timeOutClock = context.system.scheduler.scheduleOnce(timeout seconds, self, ResourceAllocationTimeOut(session)) + timeOutClock = context.system.scheduler.scheduleOnce( + timeout.seconds, self, ResourceAllocationTimeOut(session)) resourceRequestor = sender master ! request case ResourceAllocated(allocations) => @@ -164,7 +175,7 @@ object ExecutorSystemScheduler { case timeout: ResourceAllocationTimeOut => if (unallocatedResource > 0) { resourceRequestor ! ResourceAllocationTimeOut(session) - //we will not receive any ResourceAllocation after timeout + // We will not receive any ResourceAllocation after timeout context.stop(self) } } @@ -175,4 +186,5 @@ object ExecutorSystemScheduler { private[ExecutorSystemScheduler] case class ResourceAllocationTimeOut(session: Session) + } \ No newline at end of file diff --git a/core/src/main/scala/io/gearpump/cluster/appmaster/MasterConnectionKeeper.scala b/core/src/main/scala/io/gearpump/cluster/appmaster/MasterConnectionKeeper.scala index 300c4ea2d..f8c850337 100644 --- a/core/src/main/scala/io/gearpump/cluster/appmaster/MasterConnectionKeeper.scala +++ b/core/src/main/scala/io/gearpump/cluster/appmaster/MasterConnectionKeeper.scala @@ -19,31 +19,27 @@ package io.gearpump.cluster.appmaster import java.util.concurrent.TimeUnit +import scala.concurrent.duration.FiniteDuration import akka.actor._ -import io.gearpump.cluster.master.MasterProxy.MasterRestarted + import io.gearpump.cluster.AppMasterToMaster.RegisterAppMaster import io.gearpump.cluster.MasterToAppMaster.AppMasterRegistered import io.gearpump.cluster.appmaster.MasterConnectionKeeper.AppMasterRegisterTimeout import io.gearpump.cluster.appmaster.MasterConnectionKeeper.MasterConnectionStatus.{MasterConnected, MasterStopped} -import io.gearpump.cluster.master.MasterProxy.WatchMaster +import io.gearpump.cluster.master.MasterProxy.{MasterRestarted, WatchMaster} import io.gearpump.util.LogUtil -import scala.concurrent.duration.FiniteDuration - /** - * This will watch the liveness of Master. - * When Master is restarted, it will send RegisterAppMaster to the new Master instance. - * If Master is stopped, it will send the MasterConnectionStatus to listener + * Watches the liveness of Master. * - * please use MasterConnectionKeeper.props() to construct this actor + * When Master is restarted, it sends RegisterAppMaster to the new Master instance. + * If Master is stopped, it sends the MasterConnectionStatus to listener * - * @param register - * @param masterProxy - * @param masterStatusListener + * please use MasterConnectionKeeper.props() to construct this actor */ private[appmaster] -class MasterConnectionKeeper ( +class MasterConnectionKeeper( register: RegisterAppMaster, masterProxy: ActorRef, masterStatusListener: ActorRef) extends Actor { @@ -52,12 +48,13 @@ class MasterConnectionKeeper ( private val LOG = LogUtil.getLogger(getClass) private var master: ActorRef = null - //Subscribe self to masterProxy, + // Subscribe self to masterProxy, masterProxy ! WatchMaster(self) def registerAppMaster: Cancellable = { masterProxy ! register - context.system.scheduler.scheduleOnce(FiniteDuration(30, TimeUnit.SECONDS), self, AppMasterRegisterTimeout) + context.system.scheduler.scheduleOnce(FiniteDuration(30, TimeUnit.SECONDS), + self, AppMasterRegisterTimeout) } context.become(waitMasterToConfirm(registerAppMaster)) @@ -87,10 +84,15 @@ class MasterConnectionKeeper ( } private[appmaster] object MasterConnectionKeeper { + case object AppMasterRegisterTimeout object MasterConnectionStatus { + case object MasterConnected + case object MasterStopped + } + } \ No newline at end of file diff --git a/core/src/main/scala/io/gearpump/cluster/client/ClientContext.scala b/core/src/main/scala/io/gearpump/cluster/client/ClientContext.scala index 3a2868cc1..41c01d8ec 100644 --- a/core/src/main/scala/io/gearpump/cluster/client/ClientContext.scala +++ b/core/src/main/scala/io/gearpump/cluster/client/ClientContext.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,10 +19,16 @@ package io.gearpump.cluster.client import java.util.concurrent.TimeUnit +import scala.collection.JavaConverters._ +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, Future} +import scala.util.Try import akka.actor.{ActorRef, ActorSystem} import akka.util.Timeout -import com.typesafe.config.{ConfigValueFactory, Config} +import com.typesafe.config.{Config, ConfigValueFactory} +import org.slf4j.Logger + import io.gearpump.cluster.MasterToAppMaster.{AppMastersData, ReplayFromTimestampWindowTrailingEdge} import io.gearpump.cluster.MasterToClient.ReplayApplicationResult import io.gearpump.cluster._ @@ -30,17 +36,12 @@ import io.gearpump.cluster.master.MasterProxy import io.gearpump.jarstore.JarStoreService import io.gearpump.util.Constants._ import io.gearpump.util.{ActorUtil, Constants, LogUtil, Util} -import org.slf4j.Logger - -import scala.collection.JavaConversions._ -import scala.concurrent.{Await, Future} -import scala.concurrent.duration.Duration -import scala.util.Try /** * ClientContext is a user facing util to submit/manage an application. + * + * TODO: add interface to query master here */ -//TODO: add interface to query master here class ClientContext(config: Config, sys: ActorSystem, _master: ActorRef) { def this(system: ActorSystem) = { @@ -54,7 +55,7 @@ class ClientContext(config: Config, sys: ActorSystem, _master: ActorRef) { private val LOG: Logger = LogUtil.getLogger(getClass) private implicit val timeout = Timeout(5, TimeUnit.SECONDS) - implicit val system = Option(sys).getOrElse(ActorSystem(s"client${Util.randInt}" , config)) + implicit val system = Option(sys).getOrElse(ActorSystem(s"client${Util.randInt()}", config)) LOG.info(s"Starting system ${system.name}") val shouldCleanupSystem = Option(sys).isEmpty @@ -62,16 +63,18 @@ class ClientContext(config: Config, sys: ActorSystem, _master: ActorRef) { jarStoreService.init(config, system) private lazy val master: ActorRef = { - val masters = config.getStringList(Constants.GEARPUMP_CLUSTER_MASTERS).toList.flatMap(Util.parseHostList) - val master = Option(_master).getOrElse(system.actorOf(MasterProxy.props(masters), s"masterproxy${system.name}")) + val masters = config.getStringList(Constants.GEARPUMP_CLUSTER_MASTERS).asScala + .flatMap(Util.parseHostList) + val master = Option(_master).getOrElse(system.actorOf(MasterProxy.props(masters), + s"masterproxy${system.name}")) LOG.info(s"Creating master proxy ${master} for master list: $masters") master } /** - * Submit an application with default jar setting. Use java property - * "gearpump.app.jar" if defined. Otherwise, will assume the jar is on - * the target runtime classpath, and will not send it. + * Submits an application with default jar setting. Use java property "gearpump.app.jar" if + * defined. Otherwise, it assumes the jar is on the target runtime classpath, thus will + * not send the jar across the wire. */ def submit(app: Application): Int = { submit(app, System.getProperty(GEARPUMP_APP_JAR)) @@ -86,7 +89,8 @@ class ClientContext(config: Config, sys: ActorSystem, _master: ActorRef) { val appName = checkAndAddNamePrefix(app.name, System.getProperty(GEARPUMP_APP_NAME_PREFIX)) val submissionConfig = getSubmissionConfig(config) .withValue(APPLICATION_EXECUTOR_NUMBER, ConfigValueFactory.fromAnyRef(executorNum)) - val appDescription = AppDescription(appName, app.appMaster.getName, app.userConfig, submissionConfig) + val appDescription = + AppDescription(appName, app.appMaster.getName, app.userConfig, submissionConfig) val appJar = Option(jar).map(loadFile) client.submitApplication(appDescription, appJar) } @@ -99,9 +103,11 @@ class ClientContext(config: Config, sys: ActorSystem, _master: ActorRef) { ClusterConfig.filterOutDefaultConfig(config) } - def replayFromTimestampWindowTrailingEdge(appId : Int): ReplayApplicationResult = { + def replayFromTimestampWindowTrailingEdge(appId: Int): ReplayApplicationResult = { import scala.concurrent.ExecutionContext.Implicits.global - val result = Await.result(ActorUtil.askAppMaster[ReplayApplicationResult](master, appId,ReplayFromTimestampWindowTrailingEdge(appId)), Duration.Inf) + val result = Await.result( + ActorUtil.askAppMaster[ReplayApplicationResult](master, + appId, ReplayFromTimestampWindowTrailingEdge(appId)), Duration.Inf) result } @@ -115,30 +121,30 @@ class ClientContext(config: Config, sys: ActorSystem, _master: ActorRef) { client.listApplications } - def shutdown(appId : Int) : Unit = { + def shutdown(appId: Int): Unit = { val client = getMasterClient client.shutdownApplication(appId) } - def resolveAppID(appId: Int) : ActorRef = { + def resolveAppID(appId: Int): ActorRef = { val client = getMasterClient client.resolveAppId(appId) } - def close() : Unit = { + def close(): Unit = { if (shouldCleanupSystem) { LOG.info(s"Shutting down system ${system.name}") - system.shutdown() + system.terminate() } } - private def loadFile(jarPath : String) : AppJar = { + private def loadFile(jarPath: String): AppJar = { val jarFile = new java.io.File(jarPath) val path = jarStoreService.copyFromLocal(jarFile) AppJar(jarFile.getName, path) } - private def checkAndAddNamePrefix(appName: String, namePrefix: String) : String = { + private def checkAndAddNamePrefix(appName: String, namePrefix: String): String = { val fullName = if (namePrefix != null && namePrefix != "") { namePrefix + "_" + appName } else { @@ -163,12 +169,17 @@ object ClientContext { def apply(): ClientContext = new ClientContext(ClusterConfig.default(), null, null) - def apply(system: ActorSystem) = new ClientContext(ClusterConfig.default(), system, null) + def apply(system: ActorSystem): ClientContext = { + new ClientContext(ClusterConfig.default(), system, null) + } - def apply(system: ActorSystem, master: ActorRef) = new ClientContext(ClusterConfig.default(), system, master) + def apply(system: ActorSystem, master: ActorRef): ClientContext = { + new ClientContext(ClusterConfig.default(), system, master) + } def apply(config: Config): ClientContext = new ClientContext(config, null, null) - def apply(config: Config, system: ActorSystem, master: ActorRef): ClientContext - = new ClientContext(config, system, master) + def apply(config: Config, system: ActorSystem, master: ActorRef): ClientContext = { + new ClientContext(config, system, master) + } } \ No newline at end of file diff --git a/core/src/main/scala/io/gearpump/cluster/client/MasterClient.scala b/core/src/main/scala/io/gearpump/cluster/client/MasterClient.scala index 5800e8dd4..9edaf46ee 100644 --- a/core/src/main/scala/io/gearpump/cluster/client/MasterClient.scala +++ b/core/src/main/scala/io/gearpump/cluster/client/MasterClient.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,31 +18,36 @@ package io.gearpump.cluster.client +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, Future} +import scala.util.{Failure, Success} + import akka.actor.ActorRef import akka.pattern.ask import akka.util.Timeout -import io.gearpump.cluster.ClientToMaster._ -import io.gearpump.cluster.MasterToAppMaster.{AppMastersData, AppMastersDataRequest, ReplayFromTimestampWindowTrailingEdge} -import io.gearpump.cluster.MasterToClient.{ReplayApplicationResult, ResolveAppIdResult, ShutdownApplicationResult, SubmitApplicationResult} -import io.gearpump.cluster.{AppJar, AppDescription} -import io.gearpump.util.{ActorUtil, Constants} -import scala.concurrent.duration.Duration -import scala.concurrent.{ExecutionContext, Await, Future} -import scala.util.{Failure, Success} +import io.gearpump.cluster.ClientToMaster._ +import io.gearpump.cluster.MasterToAppMaster.{AppMastersData, AppMastersDataRequest} +import io.gearpump.cluster.MasterToClient.{ResolveAppIdResult, ShutdownApplicationResult, SubmitApplicationResult} +import io.gearpump.cluster.{AppDescription, AppJar} /** - * Client to Master node. - * Stateless, thread safe + * Client to inter-operate with Master node. + * + * NOTE: Stateless, thread safe */ -class MasterClient(master : ActorRef, timeout: Timeout) { +class MasterClient(master: ActorRef, timeout: Timeout) { implicit val masterClientTimeout = timeout - def submitApplication(app : AppDescription, appJar: Option[AppJar]) : Int = { - val result = Await.result( (master ? SubmitApplication(app, appJar)).asInstanceOf[Future[SubmitApplicationResult]], Duration.Inf) + def submitApplication(app: AppDescription, appJar: Option[AppJar]): Int = { + val result = Await.result( + (master ? SubmitApplication(app, appJar)).asInstanceOf[Future[SubmitApplicationResult]], + Duration.Inf) val appId = result.appId match { case Success(appId) => + // scalastyle:off println Console.println(s"Submit application succeed. The application id is $appId") + // scalastyle:on println appId case Failure(ex) => throw ex } @@ -50,15 +55,18 @@ class MasterClient(master : ActorRef, timeout: Timeout) { } def resolveAppId(appId: Int): ActorRef = { - val result = Await.result((master ? ResolveAppId(appId)).asInstanceOf[Future[ResolveAppIdResult]], Duration.Inf) + val result = Await.result( + (master ? ResolveAppId(appId)).asInstanceOf[Future[ResolveAppIdResult]], Duration.Inf) result.appMaster match { case Success(appMaster) => appMaster case Failure(ex) => throw ex } } - def shutdownApplication(appId : Int) : Unit = { - val result = Await.result((master ? ShutdownApplication(appId)).asInstanceOf[Future[ShutdownApplicationResult]], Duration.Inf) + def shutdownApplication(appId: Int): Unit = { + val result = Await.result( + (master ? ShutdownApplication(appId)).asInstanceOf[Future[ShutdownApplicationResult]], + Duration.Inf) result.appId match { case Success(_) => case Failure(ex) => throw ex @@ -66,7 +74,8 @@ class MasterClient(master : ActorRef, timeout: Timeout) { } def listApplications: AppMastersData = { - val result = Await.result((master ? AppMastersDataRequest).asInstanceOf[Future[AppMastersData]], Duration.Inf) + val result = Await.result( + (master ? AppMastersDataRequest).asInstanceOf[Future[AppMastersData]], Duration.Inf) result } } \ No newline at end of file diff --git a/core/src/main/scala/io/gearpump/cluster/main/ArgumentsParser.scala b/core/src/main/scala/io/gearpump/cluster/main/ArgumentsParser.scala index f290241d4..209f831cb 100644 --- a/core/src/main/scala/io/gearpump/cluster/main/ArgumentsParser.scala +++ b/core/src/main/scala/io/gearpump/cluster/main/ArgumentsParser.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,39 +20,44 @@ package io.gearpump.cluster.main import io.gearpump.cluster.main.ArgumentsParser.Syntax -case class CLIOption[+T] (description:String = "", required: Boolean = false, defaultValue: Option[T] = None) +case class CLIOption[+T]( + description: String = "", required: Boolean = false, defaultValue: Option[T] = None) -class ParseResult(optionMap : Map[String, String], remainArguments : Array[String]) { - def getInt(key : String) = optionMap.get(key).get.toInt +class ParseResult(optionMap: Map[String, String], remainArguments: Array[String]) { + def getInt(key: String): Int = optionMap.get(key).get.toInt - def getString (key : String) = optionMap.get(key).get + def getString(key: String): String = optionMap.get(key).get - def getBoolean (key : String) = optionMap.get(key).get.toBoolean + def getBoolean(key: String): Boolean = optionMap.get(key).get.toBoolean - def exists(key : String) = !optionMap.getOrElse(key,"").isEmpty + def exists(key: String): Boolean = !(optionMap.getOrElse(key, "").isEmpty) - def remainArgs : Array[String] = this.remainArguments + def remainArgs: Array[String] = this.remainArguments } /** - * Parse command line arguments + * Parser for command line arguments + * * Grammar: -option1 value1 -option2 value3 -flag1 -flag2 remainArg1 remainArg2... */ -trait ArgumentsParser { +trait ArgumentsParser { val ignoreUnknownArgument = false - def help: Unit = { - Console.err.println(s"\nHelp: $description") + // scalastyle:off println + def help(): Unit = { + Console.println(s"\nHelp: $description") var usage = List.empty[String] - options.map(kv => if(kv._2.required) { + options.map(kv => if (kv._2.required) { usage = usage :+ s"-${kv._1} (required:${kv._2.required})${kv._2.description}" } else { - usage = usage :+ s"-${kv._1} (required:${kv._2.required}, default:${kv._2.defaultValue.getOrElse("")})${kv._2.description}" + usage = usage :+ s"-${kv._1} (required:${kv._2.required}, " + + s"default:${kv._2.defaultValue.getOrElse("")})${kv._2.description}" }) usage :+= remainArgs.map(k => s"<$k>").mkString(" ") - usage.foreach(Console.err.println(_)) + usage.foreach(Console.println(_)) } + // scalastyle:on println def parse(args: Array[String]): ParseResult = { val syntax = Syntax(options, remainArgs, ignoreUnknownArgument) @@ -60,16 +65,18 @@ trait ArgumentsParser { } val description: String = "" - val options : Array[(String, CLIOption[Any])] = Array.empty[(String, CLIOption[Any])] - val remainArgs : Array[String] = Array.empty[String] + val options: Array[(String, CLIOption[Any])] = Array.empty[(String, CLIOption[Any])] + val remainArgs: Array[String] = Array.empty[String] } object ArgumentsParser { - case class Syntax(val options: Array[(String, CLIOption[Any])], val remainArgs : Array[String], val ignoreUnknownArgument: Boolean) + case class Syntax( + val options: Array[(String, CLIOption[Any])], val remainArgs: Array[String], + val ignoreUnknownArgument: Boolean) def parse(syntax: Syntax, args: Array[String]): ParseResult = { - import syntax.{options, remainArgs, ignoreUnknownArgument} + import syntax.{ignoreUnknownArgument, options, remainArgs} var config = Map.empty[String, String] var remain = Array.empty[String] @@ -100,14 +107,16 @@ object ArgumentsParser { doParse(rest) case value :: rest => + // scalastyle:off println Console.err.println(s"Warning: get unknown argument $value, maybe it is a main class") + // scalastyle:on println remain ++= value :: rest doParse(Nil) } } doParse(args.toList) - options.foreach{pair => + options.foreach { pair => val (key, option) = pair if (!config.contains(key) && !option.required) { config += key -> option.defaultValue.getOrElse("").toString diff --git a/core/src/main/scala/io/gearpump/cluster/master/AppMasterLauncher.scala b/core/src/main/scala/io/gearpump/cluster/master/AppMasterLauncher.scala index d391bba2b..fb3e5c411 100644 --- a/core/src/main/scala/io/gearpump/cluster/master/AppMasterLauncher.scala +++ b/core/src/main/scala/io/gearpump/cluster/master/AppMasterLauncher.scala @@ -19,36 +19,36 @@ package io.gearpump.cluster.master import java.util.concurrent.{TimeUnit, TimeoutException} +import scala.collection.JavaConverters._ +import scala.concurrent.duration.Duration +import scala.util.{Failure, Success} import akka.actor.{Actor, ActorRef, Props, _} import com.typesafe.config.Config -import io.gearpump.cluster.{AppJar, AppDescription} +import org.slf4j.Logger + import io.gearpump.cluster.AppMasterToMaster.RequestResource import io.gearpump.cluster.AppMasterToWorker.{LaunchExecutor, ShutdownExecutor} import io.gearpump.cluster.MasterToAppMaster.ResourceAllocated import io.gearpump.cluster.MasterToClient.SubmitApplicationResult import io.gearpump.cluster.WorkerToAppMaster.ExecutorLaunchRejected -import io.gearpump.cluster._ -import io.gearpump.cluster.appmaster.{WorkerInfo, AppMasterRuntimeEnvironment, AppMasterRuntimeInfo} +import io.gearpump.cluster.appmaster.{AppMasterRuntimeEnvironment, AppMasterRuntimeInfo, WorkerInfo} import io.gearpump.cluster.scheduler.{Resource, ResourceAllocation, ResourceRequest} +import io.gearpump.cluster.worker.WorkerId +import io.gearpump.cluster.{AppDescription, AppJar, _} import io.gearpump.transport.HostPort import io.gearpump.util.ActorSystemBooter._ import io.gearpump.util.Constants._ import io.gearpump.util.{ActorSystemBooter, ActorUtil, LogUtil, Util} -import org.slf4j.Logger -import scala.collection.JavaConverters._ -import scala.concurrent.duration.Duration -import scala.util.{Failure, Success} -import io.gearpump.WorkerId /** * * AppMasterLauncher is a child Actor of AppManager, it is responsible * to launch the AppMaster on the cluster. */ class AppMasterLauncher( - appId : Int, executorId: Int, app : AppDescription, - jar: Option[AppJar], username : String, master : ActorRef, client: Option[ActorRef]) + appId: Int, executorId: Int, app: AppDescription, + jar: Option[AppJar], username: String, master: ActorRef, client: Option[ActorRef]) extends Actor { private val LOG: Logger = LogUtil.getLogger(getClass, app = appId) @@ -61,9 +61,9 @@ class AppMasterLauncher( LOG.info(s"Ask Master resource to start AppMaster $appId...") master ! RequestResource(appId, ResourceRequest(Resource(1), WorkerId.unspecified)) - def receive : Receive = waitForResourceAllocation + def receive: Receive = waitForResourceAllocation - def waitForResourceAllocation : Receive = { + def waitForResourceAllocation: Receive = { case ResourceAllocated(allocations) => val ResourceAllocation(resource, worker, workerId) = allocations(0) @@ -74,22 +74,27 @@ class AppMasterLauncher( val appMasterInfo = AppMasterRuntimeInfo(appId, app.name, worker, username, submissionTime, config = appMasterAkkaConfig) val workerInfo = WorkerInfo(workerId, worker) - val appMasterContext = AppMasterContext(appId, username, resource, workerInfo, jar, null, appMasterInfo) + val appMasterContext = + AppMasterContext(appId, username, resource, workerInfo, jar, null, appMasterInfo) LOG.info(s"Try to launch a executor for AppMaster on worker ${workerId} for app $appId") val name = ActorUtil.actorNameForExecutor(appId, executorId) val selfPath = ActorUtil.getFullPath(context.system, self.path) - val jvmSetting = Util.resolveJvmSetting(appMasterAkkaConfig.withFallback(systemConfig)).appMater - val executorJVM = ExecutorJVMConfig(jvmSetting.classPath ,jvmSetting.vmargs, - classOf[ActorSystemBooter].getName, Array(name, selfPath), jar, username, appMasterAkkaConfig) + val jvmSetting = + Util.resolveJvmSetting(appMasterAkkaConfig.withFallback(systemConfig)).appMater + val executorJVM = ExecutorJVMConfig(jvmSetting.classPath, jvmSetting.vmargs, + classOf[ActorSystemBooter].getName, Array(name, selfPath), jar, + username, appMasterAkkaConfig) worker ! LaunchExecutor(appId, executorId, resource, executorJVM) context.become(waitForActorSystemToStart(worker, appMasterContext, app.userConfig, resource)) } - def waitForActorSystemToStart(worker : ActorRef, appContext : AppMasterContext, user : UserConfig, resource: Resource) : Receive = { + def waitForActorSystemToStart( + worker: ActorRef, appContext: AppMasterContext, user: UserConfig, resource: Resource) + : Receive = { case ExecutorLaunchRejected(reason, ex) => - LOG.error(s"Executor Launch failed reason：$reason", ex) + LOG.error(s"Executor Launch failed reason: $reason", ex) LOG.info(s"reallocate resource $resource to start appmaster") master ! RequestResource(appId, ResourceRequest(resource, WorkerId.unspecified)) context.become(waitForResourceAllocation) @@ -99,7 +104,8 @@ class AppMasterLauncher( val masterAddress = systemConfig.getStringList(GEARPUMP_CLUSTER_MASTERS) .asScala.map(HostPort(_)).map(ActorUtil.getMasterActorPath) - sender ! CreateActor(AppMasterRuntimeEnvironment.props(masterAddress, app, appContext), s"appdaemon$appId") + sender ! CreateActor( + AppMasterRuntimeEnvironment.props(masterAddress, app, appContext), s"appdaemon$appId") import context.dispatcher val appMasterTimeout = scheduler.scheduleOnce(TIMEOUT, self, @@ -107,7 +113,7 @@ class AppMasterLauncher( context.become(waitForAppMasterToStart(worker, appMasterTimeout)) } - def waitForAppMasterToStart(worker : ActorRef, cancel: Cancellable) : Receive = { + def waitForAppMasterToStart(worker: ActorRef, cancel: Cancellable): Receive = { case ActorCreated(appMaster, _) => cancel.cancel() sender ! BindLifeCycle(appMaster) @@ -121,19 +127,21 @@ class AppMasterLauncher( context.stop(self) } - def replyToClient(result : SubmitApplicationResult) : Unit = { + def replyToClient(result: SubmitApplicationResult): Unit = { if (client.isDefined) { client.get.tell(result, master) } } } -object AppMasterLauncher extends AppMasterLauncherFactory{ - def props(appId : Int, executorId: Int, app : AppDescription, jar: Option[AppJar], username : String, master : ActorRef, client: Option[ActorRef]) = { +object AppMasterLauncher extends AppMasterLauncherFactory { + def props(appId: Int, executorId: Int, app: AppDescription, jar: Option[AppJar], + username: String, master: ActorRef, client: Option[ActorRef]): Props = { Props(new AppMasterLauncher(appId, executorId, app, jar, username, master, client)) } } trait AppMasterLauncherFactory { - def props(appId : Int, executorId: Int, app : AppDescription, jar: Option[AppJar], username : String, master : ActorRef, client: Option[ActorRef]) : Props + def props(appId: Int, executorId: Int, app: AppDescription, jar: Option[AppJar], + username: String, master: ActorRef, client: Option[ActorRef]): Props } \ No newline at end of file diff --git a/core/src/main/scala/io/gearpump/cluster/master/MasterProxy.scala b/core/src/main/scala/io/gearpump/cluster/master/MasterProxy.scala index 5d9e410c9..61d95dcc7 100644 --- a/core/src/main/scala/io/gearpump/cluster/master/MasterProxy.scala +++ b/core/src/main/scala/io/gearpump/cluster/master/MasterProxy.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,24 +18,21 @@ package io.gearpump.cluster.master +import scala.concurrent.duration.FiniteDuration import akka.actor._ -import io.gearpump.transport.HostPort -import io.gearpump.util.{ActorUtil, LogUtil} import org.slf4j.Logger -import scala.concurrent.duration.{FiniteDuration, Duration} +import io.gearpump.transport.HostPort +import io.gearpump.util.{ActorUtil, LogUtil} /** * This works with Master HA. When there are multiple Master nodes, * This will find a active one. - * - * - * @param masters */ -class MasterProxy (masters: Iterable[ActorPath], timeout: FiniteDuration) +class MasterProxy(masters: Iterable[ActorPath], timeout: FiniteDuration) extends Actor with Stash { - import MasterProxy._ + import io.gearpump.cluster.master.MasterProxy._ val LOG: Logger = LogUtil.getLogger(getClass, name = self.path.name) @@ -48,10 +45,12 @@ class MasterProxy (masters: Iterable[ActorPath], timeout: FiniteDuration) import context.dispatcher - def findMaster() = repeatActionUtil(timeout){ - contacts foreach { contact => - LOG.info(s"sending identity to $contact") - contact ! Identify(None) + def findMaster(): Cancellable = { + repeatActionUtil(timeout) { + contacts foreach { contact => + LOG.info(s"sending identity to $contact") + contact ! Identify(None) + } } } @@ -64,11 +63,11 @@ class MasterProxy (masters: Iterable[ActorPath], timeout: FiniteDuration) super.postStop() } - override def receive : Receive = { - case _=> + override def receive: Receive = { + case _ => } - def establishing(findMaster : Cancellable): Actor.Receive = { + def establishing(findMaster: Cancellable): Actor.Receive = { case ActorIdentity(_, Some(receptionist)) => context watch receptionist LOG.info("Connected to [{}]", receptionist.path) @@ -85,10 +84,10 @@ class MasterProxy (masters: Iterable[ActorPath], timeout: FiniteDuration) } def active(receptionist: ActorRef): Actor.Receive = { - case Terminated(receptionist) ⇒ + case Terminated(receptionist) => LOG.info("Lost contact with [{}], restablishing connection", receptionist) context.become(establishing(findMaster)) - case _: ActorIdentity ⇒ // ok, from previous establish, already handled + case _: ActorIdentity => // ok, from previous establish, already handled case WatchMaster(watcher) => watchers = watchers :+ watcher } @@ -99,10 +98,10 @@ class MasterProxy (masters: Iterable[ActorPath], timeout: FiniteDuration) master forward msg } - def scheduler = context.system.scheduler + def scheduler: Scheduler = context.system.scheduler import scala.concurrent.duration._ - private def repeatActionUtil(timeout: FiniteDuration)(action : => Unit) : Cancellable = { - val send = scheduler.schedule(0 seconds, 2 seconds)(action) + private def repeatActionUtil(timeout: FiniteDuration)(action: => Unit): Cancellable = { + val send = scheduler.schedule(0.seconds, 2.seconds)(action) val suicide = scheduler.scheduleOnce(timeout) { send.cancel() self ! PoisonPill @@ -128,7 +127,7 @@ object MasterProxy { case class WatchMaster(watcher: ActorRef) import scala.concurrent.duration._ - def props(masters: Iterable[HostPort], duration: FiniteDuration = 30 seconds): Props = { + def props(masters: Iterable[HostPort], duration: FiniteDuration = 30.seconds): Props = { val contacts = masters.map(ActorUtil.getMasterActorPath(_)) Props(new MasterProxy(contacts, duration)) } diff --git a/core/src/main/scala/io/gearpump/cluster/master/MasterSummary.scala b/core/src/main/scala/io/gearpump/cluster/master/MasterSummary.scala index 8847df263..0996381c5 100644 --- a/core/src/main/scala/io/gearpump/cluster/master/MasterSummary.scala +++ b/core/src/main/scala/io/gearpump/cluster/master/MasterSummary.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,17 +15,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package io.gearpump.cluster.master import io.gearpump.util.HistoryMetricsService.HistoryMetricsConfig +/** Master status. Synced means all masters are live and synced. */ object MasterStatus { type Type = String val Synced = "synced" val UnSynced = "unsynced" } - case class MasterNode(host: String, port: Int) { def toTuple: (String, Int) = { (host, port) @@ -33,18 +34,18 @@ case class MasterNode(host: String, port: Int) { } /** - * Master information for REST API call + * Master information returned for REST API call */ case class MasterSummary( - leader: MasterNode, - cluster: List[MasterNode], - aliveFor: Long, - logFile: String, - jarStore: String, - masterStatus: MasterStatus.Type, - homeDirectory: String, - activities: List[MasterActivity], - jvmName: String, - historyMetricsConfig: HistoryMetricsConfig = null) + leader: MasterNode, + cluster: List[MasterNode], + aliveFor: Long, + logFile: String, + jarStore: String, + masterStatus: MasterStatus.Type, + homeDirectory: String, + activities: List[MasterActivity], + jvmName: String, + historyMetricsConfig: HistoryMetricsConfig = null) case class MasterActivity(time: Long, event: String) \ No newline at end of file diff --git a/core/src/main/scala/io/gearpump/cluster/scheduler/Resource.scala b/core/src/main/scala/io/gearpump/cluster/scheduler/Resource.scala index da17829de..b25162e97 100644 --- a/core/src/main/scala/io/gearpump/cluster/scheduler/Resource.scala +++ b/core/src/main/scala/io/gearpump/cluster/scheduler/Resource.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,50 +15,66 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package io.gearpump.cluster.scheduler import akka.actor.ActorRef -import io.gearpump.WorkerId -case class Resource(slots : Int) { - def +(other : Resource): Resource = Resource(slots + other.slots) +import io.gearpump.cluster.worker.WorkerId + +case class Resource(slots: Int) { - def -(other : Resource): Resource = Resource(slots - other.slots) + // scalastyle:off spaces.after.plus + def +(other: Resource): Resource = Resource(slots + other.slots) + // scalastyle:on spaces.after.plus - def >(other : Resource): Boolean = slots > other.slots + def -(other: Resource): Resource = Resource(slots - other.slots) - def >=(other : Resource): Boolean = !(this < other) + def >(other: Resource): Boolean = slots > other.slots - def <(other : Resource): Boolean = slots < other.slots + def >=(other: Resource): Boolean = !(this < other) - def <=(other : Resource): Boolean = !(this > other) + def <(other: Resource): Boolean = slots < other.slots - def equals(other : Resource): Boolean = slots == other.slots + def <=(other: Resource): Boolean = !(this > other) - def isEmpty: Boolean = slots == 0 + def isEmpty: Boolean = { + slots == 0 + } } -object Priority extends Enumeration{ +/** + * Each streaming job can have a priority, the job with higher priority + * will get scheduled resource earlier than those with lower priority. + */ +object Priority extends Enumeration { type Priority = Value val LOW, NORMAL, HIGH = Value } -object Relaxation extends Enumeration{ +/** + * Relaxation.ONEWORKER means only resource (slot) from that worker will be accepted by + * the requestor application job. + */ +object Relaxation extends Enumeration { type Relaxation = Value // Option ONEWORKER allow user to schedule a task on specific worker. val ANY, ONEWORKER, SPECIFICWORKER = Value } -import Relaxation._ -import Priority._ -case class ResourceRequest(resource: Resource, workerId: WorkerId, priority: Priority = NORMAL, relaxation: Relaxation = ANY, executorNum: Int = 1) +import io.gearpump.cluster.scheduler.Priority._ +import io.gearpump.cluster.scheduler.Relaxation._ + +case class ResourceRequest( + resource: Resource, workerId: WorkerId, priority: Priority = NORMAL, + relaxation: Relaxation = ANY, executorNum: Int = 1) -case class ResourceAllocation(resource : Resource, worker : ActorRef, workerId : WorkerId) +case class ResourceAllocation(resource: Resource, worker: ActorRef, workerId: WorkerId) object Resource { - def empty = new Resource(0) + def empty: Resource = new Resource(0) - def min(res1: Resource, res2: Resource) = if (res1.slots < res2.slots) res1 else res2 + def min(res1: Resource, res2: Resource): Resource = if (res1.slots < res2.slots) res1 else res2 } diff --git a/core/src/main/scala/io/gearpump/cluster/worker/ExecutorProcessLauncher.scala b/core/src/main/scala/io/gearpump/cluster/worker/ExecutorProcessLauncher.scala index a7e7dd2bf..85814676c 100644 --- a/core/src/main/scala/io/gearpump/cluster/worker/ExecutorProcessLauncher.scala +++ b/core/src/main/scala/io/gearpump/cluster/worker/ExecutorProcessLauncher.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,37 +15,42 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package io.gearpump.cluster.worker import com.typesafe.config.Config -import io.gearpump.util.RichProcess + import io.gearpump.cluster.scheduler.Resource +import io.gearpump.util.RichProcess /** - * ExecutorProcessLauncher is used to launch a process for Executor using given parameters. - * User can implement this interface to decide the behavior of launching a process. - * Set "gearpump.worker.executor-process-launcher" to your implemented class name. - */ + * ExecutorProcessLauncher is used to launch a process for Executor using given parameters. + * + * User can implement this interface to decide the behavior of launching a process. + * Set "gearpump.worker.executor-process-launcher" to your implemented class name. + */ trait ExecutorProcessLauncher { val config: Config /** - * This function will launch a process for Executor using given parameters. - * @param appId The appId of the executor to be launched - * @param executorId The executorId of the executor to be launched - * @param resource The resource allocated for that executor - * @param options The command options - * @param classPath The classpath of the process - * @param mainClass The main class of the process - * @param arguments The rest arguments - */ - def createProcess(appId: Int, executorId:Int, resource: Resource, config: Config, options : Array[String], - classPath : Array[String], mainClass : String, arguments : Array[String]): RichProcess + * This function launches a process for Executor using given parameters. + * + * @param appId The appId of the executor to be launched + * @param executorId The executorId of the executor to be launched + * @param resource The resource allocated for that executor + * @param options The command options + * @param classPath The classpath of the process + * @param mainClass The main class of the process + * @param arguments The rest arguments + */ + def createProcess( + appId: Int, executorId: Int, resource: Resource, config: Config, options: Array[String], + classPath: Array[String], mainClass: String, arguments: Array[String]): RichProcess /** - * This function will clean resources for a launched process. - * @param appId The appId of the launched executor - * @param executorId The executorId of launched executor - */ + * This function will clean resources for a launched process. + * @param appId The appId of the launched executor + * @param executorId The executorId of launched executor + */ def cleanProcess(appId: Int, executorId: Int): Unit } diff --git a/core/src/main/scala/io/gearpump/cluster/worker/WorkerId.scala b/core/src/main/scala/io/gearpump/cluster/worker/WorkerId.scala new file mode 100644 index 000000000..24c6ad284 --- /dev/null +++ b/core/src/main/scala/io/gearpump/cluster/worker/WorkerId.scala @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.gearpump.cluster.worker + +/** + * WorkerId is used to uniquely track a worker machine. + * + * @param sessionId sessionId is assigned by Master node for easy tracking. It is possible that + * sessionId is **NOT** unique, so always use WorkerId for comparison. + * @param registerTime the timestamp when a worker node register itself to master node + */ +case class WorkerId(sessionId: Int, registerTime: Long) + +object WorkerId { + val unspecified: WorkerId = new WorkerId(-1, 0L) + + def render(workerId: WorkerId): String = { + workerId.registerTime + "_" + workerId.sessionId + } + + def parse(str: String): WorkerId = { + val pair = str.split("_") + new WorkerId(pair(1).toInt, pair(0).toLong) + } + + implicit val workerIdOrdering: Ordering[WorkerId] = { + new Ordering[WorkerId] { + + /** Compare timestamp first, then id */ + override def compare(x: WorkerId, y: WorkerId): Int = { + if (x.registerTime < y.registerTime) { + -1 + } else if (x.registerTime == y.registerTime) { + if (x.sessionId < y.sessionId) { + -1 + } else if (x.sessionId == y.sessionId) { + 0 + } else { + 1 + } + } else { + 1 + } + } + } + } +} diff --git a/core/src/main/scala/io/gearpump/cluster/worker/WorkerSummary.scala b/core/src/main/scala/io/gearpump/cluster/worker/WorkerSummary.scala index ca700dc43..cdf2d0372 100644 --- a/core/src/main/scala/io/gearpump/cluster/worker/WorkerSummary.scala +++ b/core/src/main/scala/io/gearpump/cluster/worker/WorkerSummary.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,32 +15,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.gearpump.cluster.worker -import akka.actor.ActorRef -import io.gearpump.WorkerId +package io.gearpump.cluster.worker import io.gearpump.util.HistoryMetricsService.HistoryMetricsConfig /** * Worker summary information for REST API. */ case class WorkerSummary( - workerId: WorkerId, - state: String, - actorPath: String, - aliveFor: Long, - logFile: String, - executors: Array[ExecutorSlots], - totalSlots: Int, - availableSlots: Int, - homeDirectory: String, - jvmName: String, - // Id used to uniquely identity this worker process in low level resource manager like YARN. - resourceManagerContainerId: String, - historyMetricsConfig: HistoryMetricsConfig = null) + workerId: WorkerId, + state: String, + actorPath: String, + aliveFor: Long, + logFile: String, + executors: Array[ExecutorSlots], + totalSlots: Int, + availableSlots: Int, + homeDirectory: String, + jvmName: String, + // Id used to uniquely identity this worker process in low level resource manager like YARN. + resourceManagerContainerId: String, + historyMetricsConfig: HistoryMetricsConfig = null) -object WorkerSummary{ - def empty = WorkerSummary(WorkerId.unspecified, "", "", 0L, "", Array.empty[ExecutorSlots], 0, 0, "", jvmName = "", resourceManagerContainerId = "") +object WorkerSummary { + def empty: WorkerSummary = { + WorkerSummary(WorkerId.unspecified, "", "", 0L, "", + Array.empty[ExecutorSlots], 0, 0, "", jvmName = "", resourceManagerContainerId = "") + } } case class ExecutorSlots(appId: Int, executorId: Int, slots: Int) \ No newline at end of file diff --git a/core/src/main/scala/io/gearpump/jarstore/JarStoreService.scala b/core/src/main/scala/io/gearpump/jarstore/JarStoreService.scala index ba01c947d..54d54310b 100644 --- a/core/src/main/scala/io/gearpump/jarstore/JarStoreService.scala +++ b/core/src/main/scala/io/gearpump/jarstore/JarStoreService.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,17 +15,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package io.gearpump.jarstore import java.io.File import java.net.URI import java.util.ServiceLoader +import scala.collection.JavaConverters._ -import akka.actor.{ActorSystem, ActorRefFactory} +import akka.actor.ActorSystem import com.typesafe.config.Config -import io.gearpump.util.{Constants, Util} -import scala.collection.JavaConverters._ +import io.gearpump.util.{Constants, Util} case class FilePath(path: String) @@ -39,7 +40,7 @@ trait JarStoreService { * Like "hdfs" for HDFS file system, and "file" for a local * file system. */ - val scheme : String + val scheme: String /** * Init the Jar Store. @@ -47,13 +48,14 @@ trait JarStoreService { def init(config: Config, system: ActorSystem) /** - * This function will copy the local file to the remote JarStore, called from client side. + * This function will copy the local file to the remote JarStore, called from client side. * @param localFile The local file */ def copyFromLocal(localFile: File): FilePath /** - * This function will copy the remote file to local file system, called from client side. + * This function will copy the remote file to local file system, called from client side. + * * @param localFile The destination of file path * @param remotePath The remote file path from JarStore */ @@ -64,7 +66,8 @@ object JarStoreService { /** * Get a active JarStoreService by specifying a scheme. - * Please see config [[Constants.GEARPUMP_APP_JAR_STORE_ROOT_PATH]] for more + * + * Please see config [[io.gearpump.util.Constants#GEARPUMP_APP_JAR_STORE_ROOT_PATH]] for more * information. */ def get(config: Config): JarStoreService = { diff --git a/core/src/main/scala/io/gearpump/metrics/AkkaReporter.scala b/core/src/main/scala/io/gearpump/metrics/AkkaReporter.scala index 269f8f322..3a581fb6d 100644 --- a/core/src/main/scala/io/gearpump/metrics/AkkaReporter.scala +++ b/core/src/main/scala/io/gearpump/metrics/AkkaReporter.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,15 +18,14 @@ package io.gearpump.metrics +import scala.collection.JavaConverters._ + import akka.actor.{ActorRef, ActorSystem} -import io.gearpump.metrics.Metrics.{Histogram => HistogramData, Meter => MeterData, Counter => CounterData, Gauge => GaugeData} -import io.gearpump.codahale.metrics.MetricRegistry -import akka.actor.ActorSystem -import io.gearpump.codahale.metrics.{Gauge => CodaGauge} +import io.gearpump.codahale.metrics.{Gauge => CodaGauge, MetricRegistry} +import io.gearpump.metrics.Metrics.{Counter => CounterData, Gauge => GaugeData, Histogram => HistogramData, Meter => MeterData} import io.gearpump.metrics.MetricsReporterService.ReportTo import io.gearpump.util.LogUtil -import scala.collection.JavaConverters._ /** * A reporter class for logging metrics values to a remote actor periodically @@ -34,7 +33,7 @@ import scala.collection.JavaConverters._ class AkkaReporter( system: ActorSystem, registry: MetricRegistry) - extends ReportTo{ + extends ReportTo { private val LOG = LogUtil.getLogger(getClass) LOG.info("Start Metrics AkkaReporter") @@ -57,7 +56,7 @@ class AkkaReporter( s.get95thPercentile, s.get99thPercentile, s.get999thPercentile) } - meters.entrySet().asScala.foreach{pair => + meters.entrySet().asScala.foreach { pair => val key = pair.getKey val value = pair.getValue to ! MeterData(key, @@ -67,7 +66,7 @@ class AkkaReporter( getRateUnit) } - gauges.entrySet().asScala.foreach {kv => + gauges.entrySet().asScala.foreach { kv => val value = kv.getValue.asInstanceOf[CodaGauge[Number]].getValue.longValue() to ! GaugeData(kv.getKey, value) } diff --git a/core/src/main/scala/io/gearpump/metrics/Counter.scala b/core/src/main/scala/io/gearpump/metrics/Counter.scala index e0c9c5773..70c7bae87 100644 --- a/core/src/main/scala/io/gearpump/metrics/Counter.scala +++ b/core/src/main/scala/io/gearpump/metrics/Counter.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,9 +21,9 @@ package io.gearpump.metrics import io.gearpump.codahale.metrics.{Counter => CodaHaleCounter} /** - * sampleRate: take a data point for every sampleRate... + * @see io.gearpump.codahale.metrics.Counter */ -class Counter(val name : String, counter : CodaHaleCounter, sampleRate : Int = 1) { +class Counter(val name: String, counter: CodaHaleCounter, sampleRate: Int = 1) { private var sampleCount = 0L private var toBeIncremented = 0L diff --git a/core/src/main/scala/io/gearpump/metrics/Histogram.scala b/core/src/main/scala/io/gearpump/metrics/Histogram.scala index 2ad06c32a..467305037 100644 --- a/core/src/main/scala/io/gearpump/metrics/Histogram.scala +++ b/core/src/main/scala/io/gearpump/metrics/Histogram.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,23 +21,23 @@ package io.gearpump.metrics import io.gearpump.codahale.metrics.{Histogram => CodaHaleHistogram} /** - * sampleRate: take a data point for every sampleRate... + * @see io.gearpump.codahale.metrics.Histogram */ -class Histogram(val name : String, hisgram : CodaHaleHistogram, sampleRate : Int = 1) { +class Histogram(val name: String, histogram: CodaHaleHistogram, sampleRate: Int = 1) { private var sampleCount = 0L def update(value: Long) { sampleCount += 1 - if (null != hisgram && sampleCount % sampleRate == 0) { - hisgram.update(value) + if (null != histogram && sampleCount % sampleRate == 0) { + histogram.update(value) } } - def getMean() : Double = { - hisgram.getSnapshot.getMean + def getMean(): Double = { + histogram.getSnapshot.getMean } - def getStdDev() : Double = { - hisgram.getSnapshot.getStdDev + def getStdDev(): Double = { + histogram.getSnapshot.getStdDev } } \ No newline at end of file diff --git a/core/src/main/scala/io/gearpump/metrics/JvmMetricsSet.scala b/core/src/main/scala/io/gearpump/metrics/JvmMetricsSet.scala index 606a03353..28d420a10 100644 --- a/core/src/main/scala/io/gearpump/metrics/JvmMetricsSet.scala +++ b/core/src/main/scala/io/gearpump/metrics/JvmMetricsSet.scala @@ -1,11 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.gearpump.metrics import java.util - -import io.gearpump.codahale.metrics.jvm.{ThreadStatesGaugeSet, MemoryUsageGaugeSet} -import io.gearpump.codahale.metrics.{Gauge, Metric, MetricSet} import scala.collection.JavaConverters._ +import io.gearpump.codahale.metrics.jvm.{MemoryUsageGaugeSet, ThreadStatesGaugeSet} +import io.gearpump.codahale.metrics.{Metric, MetricSet} + class JvmMetricsSet(name: String) extends MetricSet { override def getMetrics: util.Map[String, Metric] = { diff --git a/core/src/main/scala/io/gearpump/metrics/Meter.scala b/core/src/main/scala/io/gearpump/metrics/Meter.scala index 660029c99..ca79a37a3 100644 --- a/core/src/main/scala/io/gearpump/metrics/Meter.scala +++ b/core/src/main/scala/io/gearpump/metrics/Meter.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,7 +20,8 @@ package io.gearpump.metrics import io.gearpump.codahale.metrics.{Meter => CodaHaleMeter} -class Meter(val name : String, meter : CodaHaleMeter, sampleRate : Int = 1) { +/** See io.gearpump.codahale.metrics.Meter */ +class Meter(val name: String, meter: CodaHaleMeter, sampleRate: Int = 1) { private var sampleCount = 0L private var toBeMarked = 0L @@ -37,7 +38,7 @@ class Meter(val name : String, meter : CodaHaleMeter, sampleRate : Int = 1) { } } - def getOneMinuteRate() : Double = { + def getOneMinuteRate(): Double = { meter.getOneMinuteRate } } \ No newline at end of file diff --git a/core/src/main/scala/io/gearpump/metrics/Metrics.scala b/core/src/main/scala/io/gearpump/metrics/Metrics.scala index ba9a59f55..aad1af08c 100644 --- a/core/src/main/scala/io/gearpump/metrics/Metrics.scala +++ b/core/src/main/scala/io/gearpump/metrics/Metrics.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,39 +18,40 @@ package io.gearpump.metrics +import scala.collection.JavaConverters._ import akka.actor._ +import org.slf4j.Logger + import io.gearpump.codahale.metrics._ import io.gearpump.metrics import io.gearpump.util.LogUtil -import org.slf4j.Logger - -import scala.collection.JavaConverters._ +/** Metric objects registry */ class Metrics(sampleRate: Int) extends Extension { val registry = new MetricRegistry() - def meter(name : String) = { + def meter(name: String): metrics.Meter = { new metrics.Meter(name, registry.meter(name), sampleRate) } - def histogram(name : String) = { + def histogram(name: String): Histogram = { new Histogram(name, registry.histogram(name), sampleRate) } - def histogram(name : String, sampleRate: Int) = { + def histogram(name: String, sampleRate: Int): Histogram = { new Histogram(name, registry.histogram(name), sampleRate) } - def counter(name : String) = { + def counter(name: String): Counter = { new Counter(name, registry.counter(name), sampleRate) } def register(set: MetricSet): Unit = { val names = registry.getNames - val metrics = set.getMetrics.asScala.filterKeys {key => !names.contains(key)} - metrics.foreach{kv => + val metrics = set.getMetrics.asScala.filterKeys { key => !names.contains(key) } + metrics.foreach { kv => registry.register(kv._1, kv._2) } } @@ -88,10 +89,10 @@ object Metrics extends ExtensionId[Metrics] with ExtensionIdProvider { } } - case class Histogram - (name: String, mean: Double, - stddev: Double, median: Double, - p95: Double, p99: Double, p999: Double) + case class Histogram ( + name: String, mean: Double, + stddev: Double, median: Double, + p95: Double, p99: Double, p999: Double) extends MetricType case class Counter(name: String, value: Long) extends MetricType @@ -118,7 +119,7 @@ object Metrics extends ExtensionId[Metrics] with ExtensionIdProvider { override def get(system: ActorSystem): Metrics = super.get(system) - override def lookup = Metrics + override def lookup: ExtensionId[Metrics] = Metrics override def createExtension(system: ExtendedActorSystem): Metrics = { val metricsEnabled = system.settings.config.getBoolean(GEARPUMP_METRIC_ENABLED) @@ -133,27 +134,27 @@ object Metrics extends ExtensionId[Metrics] with ExtensionIdProvider { } class DummyMetrics extends Metrics(1) { - override def register(set: MetricSet) = Unit + override def register(set: MetricSet): Unit = Unit private val meter = new metrics.Meter("", null) { - override def mark() = Unit - override def mark(n: Long) = Unit + override def mark(): Unit = Unit + override def mark(n: Long): Unit = Unit override def getOneMinuteRate(): Double = 0 } private val histogram = new metrics.Histogram("", null) { - override def update(value: Long) = Unit - override def getMean() : Double = 0 - override def getStdDev() : Double = 0 + override def update(value: Long): Unit = Unit + override def getMean(): Double = 0 + override def getStdDev(): Double = 0 } private val counter = new metrics.Counter("", null) { - override def inc() = Unit - override def inc(n: Long) = Unit + override def inc(): Unit = Unit + override def inc(n: Long): Unit = Unit } - override def meter(name : String) = meter - override def histogram(name : String) = histogram - override def counter(name : String) = counter + override def meter(name: String): metrics.Meter = meter + override def histogram(name: String): metrics.Histogram = histogram + override def counter(name: String): metrics.Counter = counter } } \ No newline at end of file diff --git a/core/src/main/scala/io/gearpump/metrics/MetricsAggregator.scala b/core/src/main/scala/io/gearpump/metrics/MetricsAggregator.scala index 6c4c34fa9..f52a06074 100644 --- a/core/src/main/scala/io/gearpump/metrics/MetricsAggregator.scala +++ b/core/src/main/scala/io/gearpump/metrics/MetricsAggregator.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,18 +18,15 @@ package io.gearpump.metrics -import java.util - import io.gearpump.cluster.MasterToClient.HistoryMetricsItem /** - * Will aggregate a full set of metrics into a smaller set + * Aggregates a larger set of metrics into a smaller set * * Sub Class must implement a constructor with signature like this: - * MetricsAggregator(config: Config) - * - * + * MetricsAggregator(config: Config) */ trait MetricsAggregator { - def aggregate(options: Map[String, String], inputs: Iterator[HistoryMetricsItem]): List[HistoryMetricsItem] + def aggregate(options: Map[String, String], inputs: Iterator[HistoryMetricsItem]) + : List[HistoryMetricsItem] } \ No newline at end of file diff --git a/core/src/main/scala/io/gearpump/metrics/MetricsReporterService.scala b/core/src/main/scala/io/gearpump/metrics/MetricsReporterService.scala index c5be04114..05decdd95 100644 --- a/core/src/main/scala/io/gearpump/metrics/MetricsReporterService.scala +++ b/core/src/main/scala/io/gearpump/metrics/MetricsReporterService.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,16 +20,22 @@ package io.gearpump.metrics import java.net.InetSocketAddress import java.util.concurrent.TimeUnit +import scala.concurrent.duration._ + +import akka.actor.{Actor, ActorRef} -import akka.actor.{ActorRef, ExtendedActorSystem, Actor} -import io.gearpump.codahale.metrics.{Slf4jReporter, MetricFilter, ScheduledReporter} -import io.gearpump.codahale.metrics.graphite.{GraphiteReporter, Graphite} -import io.gearpump.metrics.Metrics.{ReportMetrics, DemandMoreMetrics} -import io.gearpump.metrics.MetricsReporterService.{ReportTo} +import io.gearpump.codahale.metrics.graphite.{Graphite, GraphiteReporter} +import io.gearpump.codahale.metrics.{MetricFilter, Slf4jReporter} +import io.gearpump.metrics.Metrics.{DemandMoreMetrics, ReportMetrics} +import io.gearpump.metrics.MetricsReporterService.ReportTo import io.gearpump.util.Constants._ import io.gearpump.util.LogUtil -import scala.concurrent.duration._ +/** + * Reports the metrics data to some where, like Ganglia, remote Akka actor, log files... + * + * @param metrics Holds a list of metrics object. + */ class MetricsReporterService(metrics: Metrics) extends Actor { private val LOG = LogUtil.getLogger(getClass) @@ -40,14 +46,15 @@ class MetricsReporterService(metrics: Metrics) extends Actor { implicit val dispatcher = context.dispatcher def receive: Receive = { + // The subscriber is demanding more messages. case DemandMoreMetrics(subscriber) => { reporter.report(subscriber) - context.system.scheduler.scheduleOnce(reportInterval milliseconds, + context.system.scheduler.scheduleOnce(reportInterval.milliseconds, subscriber, ReportMetrics) } } - def startGraphiteReporter = { + def startGraphiteReporter(): ReportTo = { val graphiteHost = system.settings.config.getString(GEARPUMP_METRIC_GRAPHITE_HOST) val graphitePort = system.settings.config.getInt(GEARPUMP_METRIC_GRAPHITE_PORT) @@ -64,7 +71,7 @@ class MetricsReporterService(metrics: Metrics) extends Actor { } } - def startSlf4jReporter = { + def startSlf4jReporter(): ReportTo = { new ReportTo { val reporter = Slf4jReporter.forRegistry(metrics.registry) .convertRatesTo(TimeUnit.SECONDS) @@ -77,7 +84,7 @@ class MetricsReporterService(metrics: Metrics) extends Actor { } } - def startAkkaReporter = { + def startAkkaReporter(): ReportTo = { new AkkaReporter(system, metrics.registry) } @@ -85,16 +92,18 @@ class MetricsReporterService(metrics: Metrics) extends Actor { val reporterType = system.settings.config.getString(GEARPUMP_METRIC_REPORTER) LOG.info(s"Metrics reporter is enabled, using $reporterType reporter") val reporter = reporterType match { - case "graphite" => startGraphiteReporter - case "logfile" => startSlf4jReporter - case "akka" => startAkkaReporter + case "graphite" => startGraphiteReporter() + case "logfile" => startSlf4jReporter() + case "akka" => startAkkaReporter() } reporter } } object MetricsReporterService { - trait ReportTo{ + + /** Target where user want to report the metrics data to */ + trait ReportTo { def report(to: ActorRef): Unit } } \ No newline at end of file diff --git a/core/src/main/scala/io/gearpump/package.scala b/core/src/main/scala/io/gearpump/package.scala index 1ed94a7a8..187765186 100644 --- a/core/src/main/scala/io/gearpump/package.scala +++ b/core/src/main/scala/io/gearpump/package.scala @@ -1,50 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io package object gearpump { type TimeStamp = Long val LatestTime = -1 - - /** - * WorkerId is used to uniquely track a worker machine. - * - * @param sessionId sessionId is assigned by Master node for easy tracking. It is possible that - * sessionId is **NOT** unique, so always use WorkerId for comparison. - * @param registerTime the timestamp when a worker node register itself to master node - */ - case class WorkerId(sessionId: Int, registerTime: Long) - - object WorkerId { - val unspecified: WorkerId = new WorkerId(-1, 0L) - - def render(workerId: WorkerId): String = { - workerId.registerTime + "_" + workerId.sessionId - } - - def parse(str: String): WorkerId = { - val pair = str.split("_") - new WorkerId(pair(1).toInt, pair(0).toLong) - } - - implicit val workerIdOrdering: Ordering[WorkerId] = { - new Ordering[WorkerId] { - - /** Compare timestamp first, then id */ - override def compare(x: WorkerId, y: WorkerId): Int = { - if (x.registerTime < y.registerTime) { - -1 - } else if (x.registerTime == y.registerTime) { - if (x.sessionId < y.sessionId) { - -1 - } else if (x.sessionId == y.sessionId) { - 0 - } else { - 1 - } - } else { - 1 - } - } - } - } - } } diff --git a/core/src/main/scala/io/gearpump/partitioner/BroadcastPartitioner.scala b/core/src/main/scala/io/gearpump/partitioner/BroadcastPartitioner.scala index dba02eed1..0b9c57e5f 100644 --- a/core/src/main/scala/io/gearpump/partitioner/BroadcastPartitioner.scala +++ b/core/src/main/scala/io/gearpump/partitioner/BroadcastPartitioner.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,11 +20,13 @@ package io.gearpump.partitioner import io.gearpump.Message +/** Used by storm module to broadcast message to all downstream tasks */ class BroadcastPartitioner extends MulticastPartitioner { private var lastPartitionNum = -1 private var partitions = Array.empty[Int] - override def getPartitions(msg: Message, partitionNum: Int, currentPartitionId: Int): Array[Int] = { + override def getPartitions( + msg: Message, partitionNum: Int, currentPartitionId: Int): Array[Int] = { if (partitionNum != lastPartitionNum) { partitions = (0 until partitionNum).toArray lastPartitionNum = partitionNum diff --git a/core/src/main/scala/io/gearpump/partitioner/CoLocationPartitioner.scala b/core/src/main/scala/io/gearpump/partitioner/CoLocationPartitioner.scala index 3ed6dd4d4..062fc1056 100644 --- a/core/src/main/scala/io/gearpump/partitioner/CoLocationPartitioner.scala +++ b/core/src/main/scala/io/gearpump/partitioner/CoLocationPartitioner.scala @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.gearpump.partitioner import io.gearpump.Message @@ -7,7 +25,7 @@ import io.gearpump.Message * And each task in current processor will co-locate with task of last processor */ class CoLocationPartitioner extends UnicastPartitioner { - override def getPartition(msg : Message, partitionNum : Int, currentPartitionId: Int) : Int = { + override def getPartition(msg: Message, partitionNum: Int, currentPartitionId: Int): Int = { currentPartitionId } } diff --git a/core/src/main/scala/io/gearpump/partitioner/HashPartitioner.scala b/core/src/main/scala/io/gearpump/partitioner/HashPartitioner.scala index 5e4c7bc2a..6ba0cd67c 100644 --- a/core/src/main/scala/io/gearpump/partitioner/HashPartitioner.scala +++ b/core/src/main/scala/io/gearpump/partitioner/HashPartitioner.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -26,7 +26,7 @@ import io.gearpump.Message * same hash code after serialization and deserialization. */ class HashPartitioner extends UnicastPartitioner { - override def getPartition(msg : Message, partitionNum : Int, currentPartitionId: Int) : Int = { + override def getPartition(msg: Message, partitionNum: Int, currentPartitionId: Int): Int = { (msg.msg.hashCode() & Integer.MAX_VALUE) % partitionNum } } diff --git a/core/src/main/scala/io/gearpump/partitioner/Partitioner.scala b/core/src/main/scala/io/gearpump/partitioner/Partitioner.scala index 6285bb768..69104c7ed 100644 --- a/core/src/main/scala/io/gearpump/partitioner/Partitioner.scala +++ b/core/src/main/scala/io/gearpump/partitioner/Partitioner.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,22 +18,34 @@ package io.gearpump.partitioner -import io.gearpump.Message import scala.reflect.ClassTag + import org.apache.commons.lang.SerializationUtils +import io.gearpump.Message + +/** + * For processor chain: A (3 tasks) {@literal ->} B (3 tasks), partitioner decide how ONE task + * of upstream processor A send to several tasks of downstream processor B. + */ sealed trait Partitioner extends Serializable +/** + * For processor chain: A (3 tasks) {@literal ->} B (3 tasks), UnicastPartitioner does + * ONE-task {@literal ->} ONE-task mapping. + */ trait UnicastPartitioner extends Partitioner { - /** + + /** + * Gets the SINGLE downstream processor task index to send message to. * - * @param msg - * @param partitionNum - * @param currentPartitionId, used when the downstream processor want to share the same - * partition id, - * @return + * @param msg Message you want to send + * @param partitionNum How many tasks does the downstream processor have. + * @param upstreamTaskIndex Upstream task's task index who trigger the getPartition() call. + * + * @return ONE task index of downstream processor. */ - def getPartition(msg: Message, partitionNum: Int, currentPartitionId: Int): Int + def getPartition(msg: Message, partitionNum: Int, upstreamTaskIndex: Int): Int def getPartition(msg: Message, partitionNum: Int): Int = { getPartition(msg, partitionNum, Partitioner.UNKNOWN_PARTITION_ID) @@ -41,14 +53,20 @@ trait UnicastPartitioner extends Partitioner { } trait MulticastPartitioner extends Partitioner { - def getPartitions(msg: Message, partitionNum: Int, currentPartitionId: Int): Array[Int] + + /** + * Gets a list of downstream processor task indexes to send message to. + * + * @param upstreamTaskIndex Current sender task's task index. + * + */ + def getPartitions(msg: Message, partitionNum: Int, upstreamTaskIndex: Int): Array[Int] def getPartitions(msg: Message, partitionNum: Int): Array[Int] = { getPartitions(msg, partitionNum, Partitioner.UNKNOWN_PARTITION_ID) } } - sealed trait PartitionerFactory { def name: String @@ -56,26 +74,32 @@ sealed trait PartitionerFactory { def partitioner: Partitioner } -class PartitionerObject(private [this] val _partitioner: Partitioner) extends PartitionerFactory with Serializable { +/** Stores the Partitioner in an object. To use it, user need to deserialize the object */ +class PartitionerObject(private[this] val _partitioner: Partitioner) + extends PartitionerFactory with Serializable { override def name: String = partitioner.getClass.getName - override def partitioner: Partitioner = SerializationUtils.clone(_partitioner).asInstanceOf[Partitioner] + override def partitioner: Partitioner = { + SerializationUtils.clone(_partitioner).asInstanceOf[Partitioner] + } } -class PartitionerByClassName(partitionerClass: String) extends PartitionerFactory with Serializable { - override def name: String = partitionerClass +/** Store the partitioner in class Name, the user need to instantiate a new class */ +class PartitionerByClassName(partitionerClass: String) + extends PartitionerFactory with Serializable { - override def partitioner: Partitioner = Class.forName(partitionerClass).newInstance().asInstanceOf[Partitioner] + override def name: String = partitionerClass + override def partitioner: Partitioner = { + Class.forName(partitionerClass).newInstance().asInstanceOf[Partitioner] + } } - /** - * @param partitionerFactory + * @param partitionerFactory How we construct a Partitioner. */ case class PartitionerDescription(partitionerFactory: PartitionerFactory) - object Partitioner { val UNKNOWN_PARTITION_ID = -1 diff --git a/core/src/main/scala/io/gearpump/partitioner/ShuffleGroupingPartitioner.scala b/core/src/main/scala/io/gearpump/partitioner/ShuffleGroupingPartitioner.scala index 035c9d203..ff962fac9 100644 --- a/core/src/main/scala/io/gearpump/partitioner/ShuffleGroupingPartitioner.scala +++ b/core/src/main/scala/io/gearpump/partitioner/ShuffleGroupingPartitioner.scala @@ -15,11 +15,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package io.gearpump.partitioner -import io.gearpump.Message import scala.util.Random +import io.gearpump.Message + /** * The idea of ShuffleGroupingPartitioner is derived from Storm. * Messages are randomly distributed across the downstream's tasks in a way such that diff --git a/core/src/main/scala/io/gearpump/partitioner/ShufflePartitioner.scala b/core/src/main/scala/io/gearpump/partitioner/ShufflePartitioner.scala index d0d3c39d9..6b3c26ee9 100644 --- a/core/src/main/scala/io/gearpump/partitioner/ShufflePartitioner.scala +++ b/core/src/main/scala/io/gearpump/partitioner/ShufflePartitioner.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -23,17 +23,16 @@ import java.util.Random import io.gearpump.Message /** - * Round Robin partition the data. + * Round Robin partition the data to downstream processor tasks. */ class ShufflePartitioner extends UnicastPartitioner { private var seed = 0 private var count = 0 - - override def getPartition(msg : Message, partitionNum : Int, currentPartitionId: Int) : Int = { + override def getPartition(msg: Message, partitionNum: Int, currentPartitionId: Int): Int = { if (seed == 0) { - seed = newSeed + seed = newSeed() } val result = ((count + seed) & Integer.MAX_VALUE) % partitionNum @@ -41,5 +40,5 @@ class ShufflePartitioner extends UnicastPartitioner { result } - def newSeed = new Random().nextInt() + private def newSeed(): Int = new Random().nextInt() } diff --git a/core/src/main/scala/io/gearpump/security/Authenticator.scala b/core/src/main/scala/io/gearpump/security/Authenticator.scala index afb13a93d..73bc8e1a2 100644 --- a/core/src/main/scala/io/gearpump/security/Authenticator.scala +++ b/core/src/main/scala/io/gearpump/security/Authenticator.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,23 +17,21 @@ */ package io.gearpump.security -import io.gearpump.security.Authenticator.AuthenticationResult - import scala.concurrent.{ExecutionContext, Future} +import io.gearpump.security.Authenticator.AuthenticationResult /** * Authenticator for UI dashboard. * * Sub Class must implement a constructor with signature like this: - * this(config: Config) - * + * this(config: Config) */ trait Authenticator { - // TODO: Change the signature to return more attributes of user - // credentials... - def authenticate(user: String, password: String, ec: ExecutionContext): Future[AuthenticationResult] + // TODO: Change the signature to return more attributes of user credentials... + def authenticate( + user: String, password: String, ec: ExecutionContext): Future[AuthenticationResult] } object Authenticator { @@ -45,22 +43,25 @@ object Authenticator { def permissionLevel: Int } - val UnAuthenticated = new AuthenticationResult{ + val UnAuthenticated = new AuthenticationResult { override val authenticated = false override val permissionLevel = -1 } - val Guest = new AuthenticationResult{ + /** Guest can view but have no permission to submit app or write */ + val Guest = new AuthenticationResult { override val authenticated = true override val permissionLevel = 1000 } - val User = new AuthenticationResult{ + /** User can submit app, kill app, but have no permission to add or remote machines */ + val User = new AuthenticationResult { override val authenticated = true override val permissionLevel = 1000 + Guest.permissionLevel } - val Admin = new AuthenticationResult{ + /** Super user */ + val Admin = new AuthenticationResult { override val authenticated = true override val permissionLevel = 1000 + User.permissionLevel } diff --git a/core/src/main/scala/io/gearpump/security/ConfigFileBasedAuthenticator.scala b/core/src/main/scala/io/gearpump/security/ConfigFileBasedAuthenticator.scala index 3ecfd8877..0743a3fac 100644 --- a/core/src/main/scala/io/gearpump/security/ConfigFileBasedAuthenticator.scala +++ b/core/src/main/scala/io/gearpump/security/ConfigFileBasedAuthenticator.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,10 +18,12 @@ package io.gearpump.security +import scala.concurrent.{ExecutionContext, Future} + +import com.typesafe.config.Config + import io.gearpump.security.Authenticator.AuthenticationResult import io.gearpump.security.ConfigFileBasedAuthenticator._ -import com.typesafe.config.Config -import scala.concurrent.{ExecutionContext, Future} object ConfigFileBasedAuthenticator { @@ -30,7 +32,9 @@ object ConfigFileBasedAuthenticator { private val USERS = ROOT + "." + "users" private val GUESTS = ROOT + "." + "guests" - private case class Credentials(admins: Map[String, String], users: Map[String, String], guests: Map[String, String]) { + private case class Credentials( + admins: Map[String, String], users: Map[String, String], guests: Map[String, String]) { + def verify(user: String, password: String): AuthenticationResult = { if (admins.contains(user)) { if (verify(user, password, admins)) { @@ -70,26 +74,32 @@ object ConfigFileBasedAuthenticator { * users have limited permission to submit an application and etc.. * guests can not submit/kill applications, but can view the application status. * - * see conf/gear.conf section gearpump.ui-security.config-file-based-authenticator to find information - * about how to configure this authenticator. + * see conf/gear.conf section gearpump.ui-security.config-file-based-authenticator to find + * information about how to configure this authenticator. * * [Security consideration] - * It will keep one-way sha1 digest of password instead of password itself. The original password is NOT - * kept in any way, so generally it is safe. + * It will keep one-way sha1 digest of password instead of password itself. The original password is + * NOT kept in any way, so generally it is safe. * - * digesting flow (from original password to digest): - * random salt byte array of length 8 -> byte array of (salt + sha1(salt, password)) -> base64Encode * - * verification user input password with stored digest: - * base64Decode -> extract salt -> do sha1(salt, password) -> generate digest: salt + sha1 -> - * compare the generated digest with the stored digest. + * digesting flow (from original password to digest): + * {{{ + * random salt byte array of length 8 -> byte array of (salt + sha1(salt, password)) -> + * base64Encode. + * }}} * + * Verification user input password with stored digest: + * {{{ + * base64Decode -> extract salt -> do sha1(salt, password) -> generate digest: + * salt + sha1 -> compare the generated digest with the stored digest. + * }}} */ class ConfigFileBasedAuthenticator(config: Config) extends Authenticator { private val credentials = loadCredentials(config) - override def authenticate(user: String, password: String, ec: ExecutionContext): Future[AuthenticationResult] = { + override def authenticate(user: String, password: String, ec: ExecutionContext) + : Future[AuthenticationResult] = { implicit val ctx = ec Future { credentials.verify(user, password) @@ -97,13 +107,13 @@ class ConfigFileBasedAuthenticator(config: Config) extends Authenticator { } private def loadCredentials(config: Config): Credentials = { - val admins = configToMap(config, ADMINS) - val users = configToMap(config, USERS) + val admins = configToMap(config, ADMINS) + val users = configToMap(config, USERS) val guests = configToMap(config, GUESTS) new Credentials(admins, users, guests) } - private def configToMap(config : Config, path: String) = { + private def configToMap(config: Config, path: String) = { import scala.collection.JavaConverters._ config.getConfig(path).root.unwrapped.asScala.toMap map { case (k, v) => k -> v.toString } } diff --git a/core/src/main/scala/io/gearpump/security/PasswordUtil.scala b/core/src/main/scala/io/gearpump/security/PasswordUtil.scala index f8eafddc6..9bf40d2aa 100644 --- a/core/src/main/scala/io/gearpump/security/PasswordUtil.scala +++ b/core/src/main/scala/io/gearpump/security/PasswordUtil.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,9 +19,10 @@ package io.gearpump.security import java.security.MessageDigest -import sun.misc.{BASE64Decoder, BASE64Encoder} import scala.util.Try +import sun.misc.{BASE64Decoder, BASE64Encoder} + /** * Util to verify whether user input password is valid or not. * It use sha1 to do the digesting. @@ -30,9 +31,11 @@ object PasswordUtil { private val SALT_LENGTH = 8 /** - * verification user input password with stored digest: + * Verifies user input password with stored digest: + * {{{ * base64Decode -> extract salt -> do sha1(salt, password) -> * generate digest: salt + sha1 -> compare the generated digest with the stored digest. + * }}} */ def verify(password: String, stored: String): Boolean = { Try { @@ -45,7 +48,10 @@ object PasswordUtil { } /** * digesting flow (from original password to digest): - * random salt byte array of length 8 -> byte array of (salt + sha1(salt, password)) -> base64Encode + * {{{ + * random salt byte array of length 8 -> + * byte array of (salt + sha1(salt, password)) -> base64Encode + * }}} */ def hash(password: String): String = { // Salt generation 64 bits long @@ -66,8 +72,8 @@ object PasswordUtil { } private def base64Encode(data: Array[Byte]): String = { - val endecoder = new BASE64Encoder() - endecoder.encode(data) + val endecoder = new BASE64Encoder() + endecoder.encode(data) } private def base64Decode(data: String): Array[Byte] = { @@ -75,13 +81,14 @@ object PasswordUtil { decoder.decodeBuffer(data) } - private def help = { + // scalastyle:off println + private def help() = { Console.println("usage: gear io.gearpump.security.PasswordUtil -password ") } def main(args: Array[String]): Unit = { if (args.length != 2 || args(0) != "-password") { - help + help() } else { val pass = args(1) val result = hash(pass) @@ -90,4 +97,5 @@ object PasswordUtil { Console.println(result) } } + // scalastyle:on println } \ No newline at end of file diff --git a/core/src/main/scala/io/gearpump/serializer/FastKryoSerializationFramework.scala b/core/src/main/scala/io/gearpump/serializer/FastKryoSerializationFramework.scala index a69ab43db..cb9d563ef 100644 --- a/core/src/main/scala/io/gearpump/serializer/FastKryoSerializationFramework.scala +++ b/core/src/main/scala/io/gearpump/serializer/FastKryoSerializationFramework.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,12 +15,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package io.gearpump.serializer import akka.actor.ExtendedActorSystem + import io.gearpump.cluster.UserConfig -class FastKryoSerializationFramework extends SerializationFramework{ +/** + * A build-in serializer framework using kryo + * + * NOTE: The Kryo here is a shaded version by Gearpump + */ +class FastKryoSerializationFramework extends SerializationFramework { private var system: ExtendedActorSystem = null private lazy val pool = new ThreadLocal[Serializer]() { diff --git a/core/src/main/scala/io/gearpump/serializer/FastKryoSerializer.scala b/core/src/main/scala/io/gearpump/serializer/FastKryoSerializer.scala index 817cd84c0..57b7b5ea9 100644 --- a/core/src/main/scala/io/gearpump/serializer/FastKryoSerializer.scala +++ b/core/src/main/scala/io/gearpump/serializer/FastKryoSerializer.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,13 +19,14 @@ package io.gearpump.serializer import akka.actor.ExtendedActorSystem + import io.gearpump.esotericsoftware.kryo.Kryo.DefaultInstantiatorStrategy +import io.gearpump.objenesis.strategy.StdInstantiatorStrategy import io.gearpump.romix.serialization.kryo.KryoSerializerWrapper import io.gearpump.serializer.FastKryoSerializer.KryoSerializationException import io.gearpump.util.LogUtil -import io.gearpump.objenesis.strategy.StdInstantiatorStrategy -class FastKryoSerializer(system: ExtendedActorSystem) extends Serializer{ +class FastKryoSerializer(system: ExtendedActorSystem) extends Serializer { private val LOG = LogUtil.getLogger(getClass) private val config = system.settings.config @@ -37,7 +38,7 @@ class FastKryoSerializer(system: ExtendedActorSystem) extends Serializer{ kryo.setInstantiatorStrategy(strategy) private val kryoClazz = new GearpumpSerialization(config).customize(kryo) - override def serialize(message: Any) : Array[Byte] = { + override def serialize(message: Any): Array[Byte] = { try { kryoSerializer.toBinary(message) } catch { @@ -72,7 +73,7 @@ class FastKryoSerializer(system: ExtendedActorSystem) extends Serializer{ } } - override def deserialize(msg : Array[Byte]): Any = { + override def deserialize(msg: Array[Byte]): Any = { kryoSerializer.fromBinary(msg) } } diff --git a/core/src/main/scala/io/gearpump/serializer/GearpumpSerialization.scala b/core/src/main/scala/io/gearpump/serializer/GearpumpSerialization.scala index 41ccaa4a9..a7eb6cf59 100644 --- a/core/src/main/scala/io/gearpump/serializer/GearpumpSerialization.scala +++ b/core/src/main/scala/io/gearpump/serializer/GearpumpSerialization.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,16 +18,17 @@ package io.gearpump.serializer -import io.gearpump.esotericsoftware.kryo.{Kryo, Serializer => KryoSerializer} import com.typesafe.config.Config -import io.gearpump.util.{Constants, LogUtil} import org.slf4j.Logger +import io.gearpump.esotericsoftware.kryo.{Kryo, Serializer => KryoSerializer} +import io.gearpump.util.{Constants, LogUtil} + class GearpumpSerialization(config: Config) { private val LOG: Logger = LogUtil.getLogger(getClass) - def customize(kryo: Kryo): Unit = { + def customize(kryo: Kryo): Unit = { val serializationMap = configToMap(config, Constants.GEARPUMP_SERIALIZERS) @@ -37,21 +38,22 @@ class GearpumpSerialization(config: Config) { if (value == null || value.isEmpty) { - //Use default serializer for this class type + // Use default serializer for this class type kryo.register(keyClass) } else { val valueClass = Class.forName(value) - val register = kryo.register(keyClass, valueClass.newInstance().asInstanceOf[KryoSerializer[_]]) + val register = kryo.register(keyClass, + valueClass.newInstance().asInstanceOf[KryoSerializer[_]]) LOG.debug(s"Registering ${keyClass}, id: ${register.getId}") } } kryo.setReferences(false) - // require the user to register the class first before using + // Requires the user to register the class first before using kryo.setRegistrationRequired(true) } - private final def configToMap(config : Config, path: String) = { + private final def configToMap(config: Config, path: String) = { import scala.collection.JavaConverters._ config.getConfig(path).root.unwrapped.asScala.toMap map { case (k, v) => k -> v.toString } } diff --git a/core/src/main/scala/io/gearpump/serializer/SerializationFramework.scala b/core/src/main/scala/io/gearpump/serializer/SerializationFramework.scala index d466ccf46..4947dcc60 100644 --- a/core/src/main/scala/io/gearpump/serializer/SerializationFramework.scala +++ b/core/src/main/scala/io/gearpump/serializer/SerializationFramework.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,15 +15,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package io.gearpump.serializer import akka.actor.ExtendedActorSystem + import io.gearpump.cluster.UserConfig /** * User are allowed to use a customized serialization framework by extending this * interface. - * */ trait SerializationFramework { def init(system: ExtendedActorSystem, config: UserConfig) diff --git a/core/src/main/scala/io/gearpump/serializer/Serializer.scala b/core/src/main/scala/io/gearpump/serializer/Serializer.scala index 02e30dbb8..ff8b147c7 100644 --- a/core/src/main/scala/io/gearpump/serializer/Serializer.scala +++ b/core/src/main/scala/io/gearpump/serializer/Serializer.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,13 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package io.gearpump.serializer /** * User defined message serializer */ trait Serializer { - def serialize(message: Any) : Array[Byte] + def serialize(message: Any): Array[Byte] - def deserialize(msg : Array[Byte]): Any + def deserialize(msg: Array[Byte]): Any } diff --git a/core/src/main/scala/io/gearpump/transport/Express.scala b/core/src/main/scala/io/gearpump/transport/Express.scala index e8f1b8e47..101b8412e 100644 --- a/core/src/main/scala/io/gearpump/transport/Express.scala +++ b/core/src/main/scala/io/gearpump/transport/Express.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,17 +18,20 @@ package io.gearpump.transport +import scala.collection.immutable.LongMap +import scala.concurrent._ + import akka.actor._ import akka.agent.Agent +import org.slf4j.Logger + import io.gearpump.transport.netty.Client.Close import io.gearpump.transport.netty.{Context, TaskMessage} import io.gearpump.util.LogUtil -import org.slf4j.Logger - -import scala.collection.immutable.LongMap -import scala.concurrent._ trait ActorLookupById { + + /** Lookup actor ref for local task actor by providing a TaskId (TaskId.toLong) */ def lookupLocalActor(id: Long): Option[ActorRef] } @@ -40,8 +43,9 @@ trait ActorLookupById { */ class Express(val system: ExtendedActorSystem) extends Extension with ActorLookupById { - import io.gearpump.transport.Express._ import system.dispatcher + + import io.gearpump.transport.Express._ val localActorMap = Agent(LongMap.empty[ActorRef]) val remoteAddressMap = Agent(Map.empty[Long, HostPort]) @@ -59,15 +63,16 @@ class Express(val system: ExtendedActorSystem) extends Extension with ActorLooku LOG.info(s"binding to netty server $localHost") system.registerOnTermination(new Runnable { - override def run = context.close + override def run(): Unit = context.close() }) (context, serverPort, localHost) } - def unregisterLocalActor(id : Long) : Unit = { + def unregisterLocalActor(id: Long): Unit = { localActorMap.sendOff(_ - id) } + /** Start Netty client actors to connect to remote machines */ def startClients(hostPorts: Set[HostPort]): Future[Map[HostPort, ActorRef]] = { val clientsToClose = remoteClientMap.get().filterKeys(!hostPorts.contains(_)).keySet closeClients(clientsToClose) @@ -85,7 +90,7 @@ class Express(val system: ExtendedActorSystem) extends Extension with ActorLooku def closeClients(hostPorts: Set[HostPort]): Future[Map[HostPort, ActorRef]] = { remoteClientMap.alter { map => - map.filterKeys(hostPorts.contains).foreach{ hostAndClient => + map.filterKeys(hostPorts.contains).foreach { hostAndClient => val (_, client) = hostAndClient client ! Close } @@ -93,36 +98,38 @@ class Express(val system: ExtendedActorSystem) extends Extension with ActorLooku } } - def registerLocalActor(id : Long, actor: ActorRef): Unit = { + def registerLocalActor(id: Long, actor: ActorRef): Unit = { LOG.info(s"RegisterLocalActor: $id, actor: ${actor.path.name}") init localActorMap.sendOff(_ + (id -> actor)) } - def lookupLocalActor(id: Long) = localActorMap.get().get(id) + def lookupLocalActor(id: Long): Option[ActorRef] = localActorMap.get().get(id) - def lookupRemoteAddress(id : Long) = remoteAddressMap.get().get(id) + def lookupRemoteAddress(id: Long): Option[HostPort] = remoteAddressMap.get().get(id) - //transport to remote address + /** Send message to remote task */ def transport(taskMessage: TaskMessage, remote: HostPort): Unit = { val remoteClient = remoteClientMap.get.get(remote) if (remoteClient.isDefined) { remoteClient.get.tell(taskMessage, Actor.noSender) } else { - val errorMsg = s"Clients has not been launched properly before transporting messages, the destination is $remote" + val errorMsg = s"Clients has not been launched properly before transporting messages, " + + s"the destination is $remote" LOG.error(errorMsg) throw new Exception(errorMsg) } } } +/** A customized transport layer by using Akka extension */ object Express extends ExtensionId[Express] with ExtensionIdProvider { val LOG: Logger = LogUtil.getLogger(getClass) override def get(system: ActorSystem): Express = super.get(system) - override def lookup = Express + override def lookup: ExtensionId[Express] = Express override def createExtension(system: ExtendedActorSystem): Express = new Express(system) } \ No newline at end of file diff --git a/core/src/main/scala/io/gearpump/transport/HostPort.scala b/core/src/main/scala/io/gearpump/transport/HostPort.scala index 72da20335..40c43423d 100644 --- a/core/src/main/scala/io/gearpump/transport/HostPort.scala +++ b/core/src/main/scala/io/gearpump/transport/HostPort.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -25,7 +25,7 @@ case class HostPort(host: String, port: Int) { } object HostPort { - def apply(address : String) : HostPort = { + def apply(address: String): HostPort = { val hostAndPort = address.split(":") new HostPort(hostAndPort(0), hostAndPort(1).toInt) } diff --git a/core/src/main/scala/io/gearpump/transport/netty/Client.scala b/core/src/main/scala/io/gearpump/transport/netty/Client.scala index 2ebf09d4f..d5960adb0 100644 --- a/core/src/main/scala/io/gearpump/transport/netty/Client.scala +++ b/core/src/main/scala/io/gearpump/transport/netty/Client.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -23,18 +23,22 @@ import java.nio.channels.ClosedChannelException import java.util import java.util.Random import java.util.concurrent.TimeUnit +import scala.concurrent.duration.FiniteDuration +import scala.language.implicitConversions import akka.actor.Actor -import io.gearpump.transport.HostPort -import io.gearpump.util.LogUtil import org.jboss.netty.bootstrap.ClientBootstrap import org.jboss.netty.channel._ import org.slf4j.Logger -import scala.concurrent.duration.FiniteDuration -import scala.language.implicitConversions +import io.gearpump.transport.HostPort +import io.gearpump.util.LogUtil -class Client(conf: NettyConfig, factory: ChannelFactory, hostPort : HostPort) extends Actor { +/** + * Netty Client implemented as an actor, on the other side, there is a netty server Actor. + * All messages sent to this actor will be forwarded to remote machine. + */ +class Client(conf: NettyConfig, factory: ChannelFactory, hostPort: HostPort) extends Actor { import io.gearpump.transport.netty.Client._ val name = s"netty-client-$hostPort" @@ -42,7 +46,7 @@ class Client(conf: NettyConfig, factory: ChannelFactory, hostPort : HostPort) ex private final var bootstrap: ClientBootstrap = null private final val random: Random = new Random private val serializer = conf.newTransportSerializer - private var channel : Channel = null + private var channel: Channel = null var batch = new util.ArrayList[TaskMessage] @@ -52,25 +56,26 @@ class Client(conf: NettyConfig, factory: ChannelFactory, hostPort : HostPort) ex self ! Connect(0) } - def receive = messageHandler orElse connectionHandler + def receive: Receive = messageHandler orElse connectionHandler - def messageHandler : Receive = { + def messageHandler: Receive = { case msg: TaskMessage => batch.add(msg) - case flush @ Flush(flushChannel) => + case flush@Flush(flushChannel) => if (channel != flushChannel) { - Unit //Drop, as it belong to old channel flush message + Unit // Drop, as it belong to old channel flush message } else if (batch.size > 0 && flushChannel.isWritable) { send(flushChannel, batch.iterator) batch.clear() self ! flush } else { import context.dispatcher - context.system.scheduler.scheduleOnce(new FiniteDuration(conf.flushCheckInterval, TimeUnit.MILLISECONDS), self, flush) + context.system.scheduler.scheduleOnce( + new FiniteDuration(conf.flushCheckInterval, TimeUnit.MILLISECONDS), self, flush) } } - def connectionHandler : Receive = { + def connectionHandler: Receive = { case ChannelReady(channel) => this.channel = channel self ! Flush(channel) @@ -90,12 +95,12 @@ class Client(conf: NettyConfig, factory: ChannelFactory, hostPort : HostPort) ex context.become(closed) } - def closed : Receive = { - case msg : AnyRef => + def closed: Receive = { + case msg: AnyRef => LOG.error(s"This client $name is closed, drop any message ${msg.getClass.getSimpleName}...") } - private def connect(tries: Int) : Unit = { + private def connect(tries: Int): Unit = { LOG.info(s"netty client try to connect to $name, tries: $tries") if (tries <= conf.max_retries) { val remote_addr = new InetSocketAddress(hostPort.host, hostPort.port) @@ -107,7 +112,9 @@ class Client(conf: NettyConfig, factory: ChannelFactory, hostPort : HostPort) ex LOG.error(s"failed to connect to $name, reason: ${ex.getMessage}, class: ${ex.getClass}") current.close() import context.dispatcher - context.system.scheduler.scheduleOnce(new FiniteDuration(getSleepTimeMs(tries), TimeUnit.MILLISECONDS), self, Connect(tries + 1)) + context.system.scheduler.scheduleOnce( + new FiniteDuration( + getSleepTimeMs(tries), TimeUnit.MILLISECONDS), self, Connect(tries + 1)) } } else { LOG.error(s"fail to connect to a remote host $name after retied $tries ...") @@ -144,7 +151,7 @@ class Client(conf: NettyConfig, factory: ChannelFactory, hostPort : HostPort) ex batch = null } - override def postStop() = { + override def postStop(): Unit = { close() } @@ -154,9 +161,10 @@ class Client(conf: NettyConfig, factory: ChannelFactory, hostPort : HostPort) ex if (channel.isOpen) { channel.close } - LOG.error(s"failed to send requests to ${channel.getRemoteAddress} ${ex.getClass.getSimpleName}") + LOG.error(s"failed to send requests " + + s"to ${channel.getRemoteAddress} ${ex.getClass.getSimpleName}") if (!ex.isInstanceOf[ClosedChannelException]) { - LOG.error(ex.getMessage, ex) + LOG.error(ex.getMessage, ex) } self ! CompareAndReconnectIfEqual(channel) } @@ -175,18 +183,17 @@ class Client(conf: NettyConfig, factory: ChannelFactory, hostPort : HostPort) ex private def isChannelWritable = (null != channel) && channel.isWritable } - object Client { val LOG: Logger = LogUtil.getLogger(getClass) - //Reconnect if current channel equals channel + // Reconnect if current channel equals channel case class CompareAndReconnectIfEqual(channel: Channel) case class Connect(tries: Int) - case class ChannelReady(chanel : Channel) + case class ChannelReady(chanel: Channel) case object Close - case class Flush(channel : Channel) + case class Flush(channel: Channel) class ClientErrorHandler(name: String) extends SimpleChannelUpstreamHandler { @@ -210,7 +217,9 @@ object Client { } } - implicit def channelFutureToChannelFutureOps(channel: ChannelFuture): ChannelFutureOps = new ChannelFutureOps(channel) + implicit def channelFutureToChannelFutureOps(channel: ChannelFuture): ChannelFutureOps = { + new ChannelFutureOps(channel) + } class ChannelFutureOps(channelFuture: ChannelFuture) { diff --git a/core/src/main/scala/io/gearpump/transport/netty/Context.scala b/core/src/main/scala/io/gearpump/transport/netty/Context.scala index fac190e6d..9a9ee29f4 100644 --- a/core/src/main/scala/io/gearpump/transport/netty/Context.scala +++ b/core/src/main/scala/io/gearpump/transport/netty/Context.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,30 +20,26 @@ package io.gearpump.transport.netty import java.io.Closeable import java.util.concurrent._ +import scala.collection.JavaConverters._ import akka.actor.{ActorRef, ActorSystem, Props} import com.typesafe.config.Config -import io.gearpump.transport.netty.Server.ServerPipelineFactory -import io.gearpump.transport.{ActorLookupById, HostPort} -import io.gearpump.util.{Constants, LogUtil} import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory import org.slf4j.Logger -import scala.collection.JavaConversions._ -import scala.language.implicitConversions +import io.gearpump.transport.netty.Server.ServerPipelineFactory +import io.gearpump.transport.{ActorLookupById, HostPort} +import io.gearpump.util.{Constants, LogUtil} object Context { private final val LOG: Logger = LogUtil.getLogger(getClass) - - implicit def toCloseable(fun : () => Any) = new Closeable { - override def close = fun() - } } -class Context(system : ActorSystem, conf: NettyConfig) extends IContext { -import io.gearpump.transport.netty.Context._ +/** Netty Context */ +class Context(system: ActorSystem, conf: NettyConfig) extends IContext { + import io.gearpump.transport.netty.Context._ - def this(system : ActorSystem, conf : Config) { + def this(system: ActorSystem, conf: Config) { this(system, new NettyConfig(conf)) } @@ -54,52 +50,64 @@ import io.gearpump.transport.netty.Context._ private lazy val clientChannelFactory: NioClientSocketChannelFactory = { val bossFactory: ThreadFactory = new NettyRenameThreadFactory("client" + "-boss") val workerFactory: ThreadFactory = new NettyRenameThreadFactory("client" + "-worker") - val channelFactory = new NioClientSocketChannelFactory(Executors.newCachedThreadPool(bossFactory), Executors.newCachedThreadPool(workerFactory), maxWorkers) - closeHandler.add { ()=> - - LOG.info("Closing all client resources....") - channelFactory.releaseExternalResources - } + val channelFactory = + new NioClientSocketChannelFactory( + Executors.newCachedThreadPool(bossFactory), + Executors.newCachedThreadPool(workerFactory), maxWorkers) + + closeHandler.add(new Closeable { + override def close(): Unit = { + LOG.info("Closing all client resources....") + channelFactory.releaseExternalResources + } + }) channelFactory } - - def bind(name: String, lookupActor : ActorLookupById, deserializeFlag : Boolean = true, inputPort: Int = 0): Int = { - //TODO: whether we should expose it as application config? - val server = system.actorOf(Props(classOf[Server], name, conf, lookupActor, deserializeFlag).withDispatcher(nettyDispatcher), name) + def bind( + name: String, lookupActor : ActorLookupById, deserializeFlag : Boolean = true, + inputPort: Int = 0): Int = { + // TODO: whether we should expose it as application config? + val server = system.actorOf(Props(classOf[Server], name, conf, lookupActor, + deserializeFlag).withDispatcher(nettyDispatcher), name) val (port, channel) = NettyUtil.newNettyServer(name, new ServerPipelineFactory(server, conf), 5242880, inputPort) val factory = channel.getFactory - closeHandler.add{ () => + closeHandler.add(new Closeable { + override def close(): Unit = { system.stop(server) channel.close() - LOG.info("Closing all server resources....") factory.releaseExternalResources } + }) port } - def connect(hostPort : HostPort) : ActorRef = { - val client = system.actorOf(Props(classOf[Client], conf, clientChannelFactory, hostPort).withDispatcher(nettyDispatcher)) - closeHandler.add { () => + def connect(hostPort: HostPort): ActorRef = { + val client = system.actorOf(Props(classOf[Client], conf, clientChannelFactory, hostPort) + .withDispatcher(nettyDispatcher)) + closeHandler.add(new Closeable { + override def close(): Unit = { + LOG.info("closing Client actor....") + system.stop(client) + } + }) - LOG.info("closing Client actor....") - system.stop(client) - } client } /** * terminate this context */ - def close { + def close(): Unit = { - LOG.info(s"Context.term, cleanup resources...., we have ${closeHandler.size()} items to close...") + LOG.info(s"Context.term, cleanup resources...., " + + s"we have ${closeHandler.size()} items to close...") - // clean up resource in reverse order so that client actor can be cleaned + // Cleans up resource in reverse order so that client actor can be cleaned // before clientChannelFactory - closeHandler.iterator().toArray.reverse.foreach(_.close()) + closeHandler.iterator().asScala.toList.reverse.foreach(_.close()) } } diff --git a/core/src/main/scala/io/gearpump/transport/netty/IContext.scala b/core/src/main/scala/io/gearpump/transport/netty/IContext.scala index e1a5141a8..56b2f7c35 100644 --- a/core/src/main/scala/io/gearpump/transport/netty/IContext.scala +++ b/core/src/main/scala/io/gearpump/transport/netty/IContext.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,24 +19,23 @@ package io.gearpump.transport.netty import akka.actor.ActorRef -import io.gearpump.transport.{HostPort, ActorLookupById} + import io.gearpump.transport.{ActorLookupById, HostPort} trait IContext { /** - * TODO: remove deserializeFlag from interface + * Create a Netty server connection. */ - def bind(name: String, lookupActor : ActorLookupById, deserializeFlag : Boolean, port: Int) : Int + def bind(name: String, lookupActor: ActorLookupById, deserializeFlag: Boolean, port: Int): Int /** - * connect to a remote host - * return a ActorRef which you can send message TaskMessage to.. + * Create a Netty client actor */ def connect(hostPort: HostPort): ActorRef /** * Close resource for this context */ - def close + def close() } \ No newline at end of file diff --git a/core/src/main/scala/io/gearpump/transport/netty/NettyConfig.scala b/core/src/main/scala/io/gearpump/transport/netty/NettyConfig.scala index ab76d9f17..a62eff5ea 100644 --- a/core/src/main/scala/io/gearpump/transport/netty/NettyConfig.scala +++ b/core/src/main/scala/io/gearpump/transport/netty/NettyConfig.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,6 +19,7 @@ package io.gearpump.transport.netty import com.typesafe.config.Config + import io.gearpump.util.Constants class NettyConfig(conf: Config) { @@ -30,6 +31,9 @@ class NettyConfig(conf: Config) { val messageBatchSize = conf.getInt(Constants.NETTY_MESSAGE_BATCH_SIZE) val flushCheckInterval = conf.getInt(Constants.NETTY_FLUSH_CHECK_INTERVAL) - def newTransportSerializer = Class.forName(conf.getString(Constants.GEARPUMP_TRANSPORT_SERIALIZER)). - newInstance().asInstanceOf[ITransportMessageSerializer] + def newTransportSerializer: ITransportMessageSerializer = { + Class.forName( + conf.getString(Constants.GEARPUMP_TRANSPORT_SERIALIZER)) + .newInstance().asInstanceOf[ITransportMessageSerializer] + } } diff --git a/core/src/main/scala/io/gearpump/transport/netty/NettyUtil.scala b/core/src/main/scala/io/gearpump/transport/netty/NettyUtil.scala index 40dc9f0f1..3e746af21 100644 --- a/core/src/main/scala/io/gearpump/transport/netty/NettyUtil.scala +++ b/core/src/main/scala/io/gearpump/transport/netty/NettyUtil.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -27,18 +27,25 @@ import org.jboss.netty.channel.{Channel, ChannelFactory, ChannelPipelineFactory} object NettyUtil { - def newNettyServer(name: String, pipelineFactory: ChannelPipelineFactory, buffer_size: Int, inputPort: Int = 0): (Int, Channel) = { + def newNettyServer( + name: String, + pipelineFactory: ChannelPipelineFactory, + buffer_size: Int, + inputPort: Int = 0): (Int, Channel) = { val bossFactory: ThreadFactory = new NettyRenameThreadFactory(name + "-boss") val workerFactory: ThreadFactory = new NettyRenameThreadFactory(name + "-worker") - val factory = new NioServerSocketChannelFactory(Executors.newCachedThreadPool(bossFactory), Executors.newCachedThreadPool(workerFactory), 1) + val factory = new NioServerSocketChannelFactory(Executors.newCachedThreadPool(bossFactory), + Executors.newCachedThreadPool(workerFactory), 1) val bootstrap = createServerBootStrap(factory, pipelineFactory, buffer_size) val channel: Channel = bootstrap.bind(new InetSocketAddress(inputPort)) - val port = channel.getLocalAddress().asInstanceOf[InetSocketAddress].getPort(); + val port = channel.getLocalAddress().asInstanceOf[InetSocketAddress].getPort() (port, channel) } - def createServerBootStrap(factory: ChannelFactory, pipelineFactory: ChannelPipelineFactory, buffer_size: Int) = { + def createServerBootStrap( + factory: ChannelFactory, pipelineFactory: ChannelPipelineFactory, buffer_size: Int) + : ServerBootstrap = { val bootstrap = new ServerBootstrap(factory) bootstrap.setOption("child.tcpNoDelay", true) bootstrap.setOption("child.receiveBufferSize", buffer_size) @@ -47,7 +54,9 @@ object NettyUtil { bootstrap } - def createClientBootStrap(factory: ChannelFactory, pipelineFactory: ChannelPipelineFactory, buffer_size: Int) = { + def createClientBootStrap( + factory: ChannelFactory, pipelineFactory: ChannelPipelineFactory, buffer_size: Int) + : ClientBootstrap = { val bootstrap = new ClientBootstrap(factory) bootstrap.setOption("tcpNoDelay", true) bootstrap.setOption("sendBufferSize", buffer_size) diff --git a/core/src/main/scala/io/gearpump/transport/netty/Server.scala b/core/src/main/scala/io/gearpump/transport/netty/Server.scala index dde08611c..9a9d79b77 100644 --- a/core/src/main/scala/io/gearpump/transport/netty/Server.scala +++ b/core/src/main/scala/io/gearpump/transport/netty/Server.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,32 +19,36 @@ package io.gearpump.transport.netty import java.util +import scala.collection.JavaConverters._ +import scala.collection.immutable.IntMap +import scala.concurrent.Future import akka.actor.{Actor, ActorContext, ActorRef, ExtendedActorSystem} -import io.gearpump.transport.ActorLookupById -import io.gearpump.util.{LogUtil, AkkaHelper} import org.jboss.netty.channel._ import org.jboss.netty.channel.group.{ChannelGroup, DefaultChannelGroup} import org.slf4j.Logger -import scala.collection.JavaConversions._ -import scala.collection.immutable.{IntMap, LongMap} -import scala.concurrent.Future +import io.gearpump.transport.ActorLookupById +import io.gearpump.util.{AkkaHelper, LogUtil} -class Server(name: String, conf: NettyConfig, lookupActor : ActorLookupById, deserializeFlag : Boolean) extends Actor { - private[netty] final val LOG: Logger = LogUtil.getLogger(getClass, context = name) +/** Netty server actor, message received will be forward to the target on the address line. */ +class Server( + name: String, conf: NettyConfig, lookupActor: ActorLookupById, deserializeFlag: Boolean) + extends Actor { + private[netty] final val LOG: Logger = LogUtil.getLogger(getClass, context = name) import io.gearpump.transport.netty.Server._ val allChannels: ChannelGroup = new DefaultChannelGroup("gearpump-server") val system = context.system.asInstanceOf[ExtendedActorSystem] - def receive = msgHandler orElse channelManager - //As we will only transfer TaskId on the wire, this object will translate taskId to or from ActorRef + def receive: Receive = msgHandler orElse channelManager + // As we will only transfer TaskId on the wire, + // this object will translate taskId to or from ActorRef private val taskIdActorRefTranslation = new TaskIdActorRefTranslation(context) - def channelManager : Receive = { + def channelManager: Receive = { case AddChannel(channel) => allChannels.add(channel) case CloseChannel(channel) => import context.dispatcher @@ -54,9 +58,9 @@ class Server(name: String, conf: NettyConfig, lookupActor : ActorLookupById, des } } - def msgHandler : Receive = { + def msgHandler: Receive = { case MsgBatch(msgs) => - msgs.groupBy(_.targetTask()).foreach { taskBatch => + msgs.asScala.groupBy(_.targetTask()).foreach { taskBatch => val (taskId, taskMessages) = taskBatch val actor = lookupActor.lookupLocalActor(taskId) @@ -69,18 +73,13 @@ class Server(name: String, conf: NettyConfig, lookupActor : ActorLookupById, des } } - override def postStop() = { + override def postStop(): Unit = { allChannels.close.awaitUninterruptibly } } object Server { - // Create a 1-1 mapping fake ActorRef for task - // The path is fake, don't use the ActorRef directly. - // As we must use actorFor() which is deprecated, - // according to the advice https://issues.scala-lang.org/browse/SI-7934, - // use a helper object to bypass this deprecation warning. class ServerPipelineFactory(server: ActorRef, conf: NettyConfig) extends ChannelPipelineFactory { def getPipeline: ChannelPipeline = { val pipeline: ChannelPipeline = Channels.pipeline @@ -114,8 +113,9 @@ object Server { class TaskIdActorRefTranslation(context: ActorContext) { private var taskIdtoActorRef = IntMap.empty[ActorRef] - def translateToActorRef(sessionId : Int): ActorRef = { - if(!taskIdtoActorRef.contains(sessionId)){ + /** 1-1 mapping from session id to fake ActorRef */ + def translateToActorRef(sessionId: Int): ActorRef = { + if (!taskIdtoActorRef.contains(sessionId)) { // A fake ActorRef for performance optimization. val actorRef = AkkaHelper.actorFor(context.system, s"/session#$sessionId") @@ -123,13 +123,12 @@ object Server { } taskIdtoActorRef.get(sessionId).get } - } case class AddChannel(channel: Channel) case class CloseChannel(channel: Channel) - case class MsgBatch(messages: Iterable[TaskMessage]) + case class MsgBatch(messages: java.lang.Iterable[TaskMessage]) } \ No newline at end of file diff --git a/core/src/main/scala/io/gearpump/util/ActorSystemBooter.scala b/core/src/main/scala/io/gearpump/util/ActorSystemBooter.scala index 71fda90bf..25a34d9e3 100644 --- a/core/src/main/scala/io/gearpump/util/ActorSystemBooter.scala +++ b/core/src/main/scala/io/gearpump/util/ActorSystemBooter.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,16 +18,17 @@ package io.gearpump.util -import java.util.concurrent.{TimeoutException, TimeUnit} +import java.util.concurrent.{TimeUnit, TimeoutException} +import scala.concurrent.Await +import scala.concurrent.duration.Duration +import scala.util.{Failure, Success, Try} import akka.actor._ import com.typesafe.config.Config -import io.gearpump.cluster.ClusterConfig -import io.gearpump.util.LogUtil.ProcessType import org.slf4j.Logger -import scala.concurrent.duration.Duration -import scala.util.{Failure, Success, Try} +import io.gearpump.cluster.ClusterConfig +import io.gearpump.util.LogUtil.ProcessType /** * ActorSystemBooter start a new JVM process to boot an actor system. @@ -35,23 +36,22 @@ import scala.util.{Failure, Success, Try} * * It send the system address to "report back actor" */ - -class ActorSystemBooter(config : Config) { +class ActorSystemBooter(config: Config) { import io.gearpump.util.ActorSystemBooter._ - def boot(name : String, reportBackActor : String) : ActorSystem = { + def boot(name: String, reportBackActor: String): ActorSystem = { val system = ActorSystem(name, config) - // daemon path: http://{system}@{ip}:{port}/daemon + // Daemon path: http://{system}@{ip}:{port}/daemon system.actorOf(Props(classOf[Daemon], name, reportBackActor), "daemon") system } } -object ActorSystemBooter { +object ActorSystemBooter { - def apply(config : Config) : ActorSystemBooter = new ActorSystemBooter(config) + def apply(config: Config): ActorSystemBooter = new ActorSystemBooter(config) - def main (args: Array[String]) { + def main(args: Array[String]) { val name = args(0) val reportBack = args(1) val config = ClusterConfig.default() @@ -59,7 +59,7 @@ object ActorSystemBooter { LogUtil.loadConfiguration(config, ProcessType.APPLICATION) val debugPort = Option(System.getProperty(Constants.GEARPUMP_REMOTE_DEBUG_PORT)) - debugPort.foreach{ port => + debugPort.foreach { port => val LOG: Logger = LogUtil.getLogger(ActorSystemBooter.getClass) LOG.info("==========================================") LOG.info("Remote debug port: " + port) @@ -69,22 +69,23 @@ object ActorSystemBooter { val system = apply(config).boot(name, reportBack) Runtime.getRuntime().addShutdownHook(new Thread() { - override def run() : Unit = { + override def run(): Unit = { val LOG: Logger = LogUtil.getLogger(ActorSystemBooter.getClass) - LOG.info("Maybe we have received a SIGINT signal from parent process, start to cleanup resources....") - system.shutdown() + LOG.info("Maybe we have received a SIGINT signal from parent process, " + + "start to cleanup resources....") + system.terminate() } - }); + }) - system.awaitTermination() + Await.result(system.whenTerminated, Duration.Inf) } - case class BindLifeCycle(actor : ActorRef) - case class CreateActor(prop : Props, name : String) - case class ActorCreated(actor : ActorRef, name : String) - case class CreateActorFailed(name : String, reason: Throwable) + case class BindLifeCycle(actor: ActorRef) + case class CreateActor(prop: Props, name: String) + case class ActorCreated(actor: ActorRef, name: String) + case class CreateActorFailed(name: String, reason: Throwable) - case class RegisterActorSystem(systemPath : String) + case class RegisterActorSystem(systemPath: String) /** * This actor system will watch for parent, @@ -95,23 +96,24 @@ object ActorSystemBooter { object RegisterActorSystemTimeOut - class Daemon(val name : String, reportBack : String) extends Actor { + class Daemon(val name: String, reportBack: String) extends Actor { val LOG: Logger = LogUtil.getLogger(getClass, context = name) val username = Option(System.getProperty(Constants.GEARPUMP_USERNAME)).getOrElse("not_defined") LOG.info(s"RegisterActorSystem to ${reportBack}, current user: $username") - + val reportBackActor = context.actorSelection(reportBack) reportBackActor ! RegisterActorSystem(ActorUtil.getSystemAddress(context.system).toString) implicit val executionContext = context.dispatcher - val timeout = context.system.scheduler.scheduleOnce(Duration(25, TimeUnit.SECONDS), self, RegisterActorSystemFailed(new TimeoutException)) + val timeout = context.system.scheduler.scheduleOnce(Duration(25, TimeUnit.SECONDS), + self, RegisterActorSystemFailed(new TimeoutException)) context.become(waitForRegisterResult) - def receive : Receive = null + def receive: Receive = null - def waitForRegisterResult : Receive = { + def waitForRegisterResult: Receive = { case ActorSystemRegistered(parent) => timeout.cancel() context.watch(parent) @@ -122,11 +124,11 @@ object ActorSystemBooter { context.stop(self) } - def waitCommand : Receive = { + def waitCommand: Receive = { case BindLifeCycle(actor) => LOG.info(s"ActorSystem $name Binding life cycle with actor: $actor") context.watch(actor) - case create @ CreateActor(props : Props, name : String) => + case create@CreateActor(props: Props, name: String) => LOG.info(s"creating actor $name") val actor = Try(context.actorOf(props, name)) actor match { @@ -142,9 +144,9 @@ object ActorSystemBooter { context.stop(self) } - override def postStop : Unit = { + override def postStop(): Unit = { LOG.info(s"ActorSystem $name is shutting down...") - context.system.shutdown() + context.system.terminate() } } } diff --git a/core/src/main/scala/io/gearpump/util/ActorUtil.scala b/core/src/main/scala/io/gearpump/util/ActorUtil.scala index cc701d84b..d5f48a7d3 100644 --- a/core/src/main/scala/io/gearpump/util/ActorUtil.scala +++ b/core/src/main/scala/io/gearpump/util/ActorUtil.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,43 +18,44 @@ package io.gearpump.util +import scala.concurrent.{ExecutionContext, Future} + import akka.actor.Actor.Receive import akka.actor._ import akka.pattern.ask -import io.gearpump.WorkerId +import org.slf4j.Logger + import io.gearpump.cluster.AppMasterToMaster.GetAllWorkers import io.gearpump.cluster.ClientToMaster.{ResolveAppId, ResolveWorkerId} import io.gearpump.cluster.MasterToAppMaster.WorkerList import io.gearpump.cluster.MasterToClient.{ResolveAppIdResult, ResolveWorkerIdResult} import io.gearpump.cluster.appmaster.ExecutorSystemScheduler.{ExecutorSystemJvmConfig, StartExecutorSystems} import io.gearpump.cluster.scheduler.{Relaxation, Resource, ResourceRequest} +import io.gearpump.cluster.worker.WorkerId import io.gearpump.transport.HostPort -import org.slf4j.Logger - -import scala.concurrent.{ExecutionContext, Future} object ActorUtil { - private val LOG: Logger = LogUtil.getLogger(getClass) + private val LOG: Logger = LogUtil.getLogger(getClass) - def getSystemAddress(system : ActorSystem) : Address = { + def getSystemAddress(system: ActorSystem): Address = { system.asInstanceOf[ExtendedActorSystem].provider.getDefaultAddress } - def getFullPath(system : ActorSystem, path : ActorPath): String = { + def getFullPath(system: ActorSystem, path: ActorPath): String = { path.toStringWithAddress(getSystemAddress(system)) } - def getHostname(actor : ActorRef): String = { + def getHostname(actor: ActorRef): String = { val path = actor.path path.address.host.getOrElse("localhost") } - def defaultMsgHandler(actor : ActorRef) : Receive = { - case msg : Any => - LOG.error(s"Cannot find a matching message, ${msg.getClass.toString}, forwarded from $actor") + def defaultMsgHandler(actor: ActorRef): Receive = { + case msg: Any => + LOG.error(s"Cannot find a matching message, ${msg.getClass.toString}, forwarded from $actor") } - def printActorSystemTree(system : ActorSystem) : Unit = { + def printActorSystemTree(system: ActorSystem): Unit = { val extendedSystem = system.asInstanceOf[ExtendedActorSystem] val clazz = system.getClass val m = clazz.getDeclaredMethod("printTree") @@ -62,9 +63,9 @@ object ActorUtil { LOG.info(m.invoke(system).asInstanceOf[String]) } - // Check whether a actor is child actor by simply examining name - //TODO: fix this, we should also check the path to root besides name - def isChildActorPath(parent : ActorRef, child : ActorRef) : Boolean = { + /** Checks whether a actor is child actor by simply examining name */ + // TODO: fix this, we should also check the path to root besides name + def isChildActorPath(parent: ActorRef, child: ActorRef): Boolean = { if (null != child) { parent.path.name == child.path.parent.name } else { @@ -72,21 +73,18 @@ object ActorUtil { } } - def actorNameForExecutor(appId : Int, executorId : Int) = "app" + appId + "-executor" + executorId + def actorNameForExecutor(appId: Int, executorId: Int): String = "app" + appId + "-executor" + + executorId - /** - * TODO: - * Currently we explicitly require the master contacts to be started with this path pattern - * 'akka.tcp://$MASTER@${master.host}:${master.port}/user/$MASTER' - * - */ + // TODO: Currently we explicitly require the master contacts to be started with this path pattern + // akka.tcp://$MASTER@${master.host}:${master.port}/user/$MASTER def getMasterActorPath(master: HostPort): ActorPath = { - import Constants.MASTER + import io.gearpump.util.Constants.MASTER ActorPath.fromString(s"akka.tcp://$MASTER@${master.host}:${master.port}/user/$MASTER") } def launchExecutorOnEachWorker(master: ActorRef, executorJvmConfig: ExecutorSystemJvmConfig, - sender: ActorRef)(implicit executor : scala.concurrent.ExecutionContext) = { + sender: ActorRef)(implicit executor: scala.concurrent.ExecutionContext): Unit = { implicit val timeout = Constants.FUTURE_TIMEOUT (master ? GetAllWorkers).asInstanceOf[Future[WorkerList]].map { list => @@ -98,10 +96,10 @@ object ActorUtil { } } - - def askAppMaster[T](master: ActorRef, appId: Int, msg: Any)(implicit ex: ExecutionContext): Future[T] = { + def askAppMaster[T](master: ActorRef, appId: Int, msg: Any)(implicit ex: ExecutionContext) + : Future[T] = { implicit val timeout = Constants.FUTURE_TIMEOUT - val appmaster = askActor[ResolveAppIdResult](master, ResolveAppId(appId)).flatMap { result => + val appmaster = askActor[ResolveAppIdResult](master, ResolveAppId(appId)).flatMap { result => if (result.appMaster.isSuccess) { Future.successful(result.appMaster.get) } else { @@ -111,15 +109,17 @@ object ActorUtil { appmaster.flatMap(askActor[T](_, msg)) } - def askWorker[T](master: ActorRef, workerId: WorkerId, msg: Any)(implicit ex: ExecutionContext): Future[T] = { + def askWorker[T](master: ActorRef, workerId: WorkerId, msg: Any)(implicit ex: ExecutionContext) + : Future[T] = { implicit val timeout = Constants.FUTURE_TIMEOUT - val worker = askActor[ResolveWorkerIdResult](master, ResolveWorkerId(workerId)).flatMap { result => - if (result.worker.isSuccess) { - Future.successful(result.worker.get) - } else { - Future.failed(result.worker.failed.get) + val worker = askActor[ResolveWorkerIdResult](master, ResolveWorkerId(workerId)) + .flatMap { result => + if (result.worker.isSuccess) { + Future.successful(result.worker.get) + } else { + Future.failed(result.worker.failed.get) + } } - } worker.flatMap(askActor[T](_, msg)) } diff --git a/core/src/main/scala/io/gearpump/util/AkkaApp.scala b/core/src/main/scala/io/gearpump/util/AkkaApp.scala index aee02d558..2b0bf6179 100644 --- a/core/src/main/scala/io/gearpump/util/AkkaApp.scala +++ b/core/src/main/scala/io/gearpump/util/AkkaApp.scala @@ -1,9 +1,26 @@ -package io.gearpump.util +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -import io.gearpump.cluster.ClusterConfig +package io.gearpump.util import scala.util.Try +import io.gearpump.cluster.ClusterConfig /** * A Main class helper to load Akka configuration automatically. @@ -14,7 +31,7 @@ trait AkkaApp { def main(akkaConf: Config, args: Array[String]): Unit - def help: Unit + def help(): Unit protected def akkaConfig: Config = { ClusterConfig.default() @@ -23,6 +40,6 @@ trait AkkaApp { def main(args: Array[String]): Unit = { Try { main(akkaConfig, args) - }.failed.foreach{ex => help; throw ex} + }.failed.foreach { ex => help(); throw ex } } } diff --git a/core/src/main/scala/io/gearpump/util/Constants.scala b/core/src/main/scala/io/gearpump/util/Constants.scala index a4fb54501..65b70230b 100644 --- a/core/src/main/scala/io/gearpump/util/Constants.scala +++ b/core/src/main/scala/io/gearpump/util/Constants.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -44,7 +44,8 @@ object Constants { val GEARPUMP_TASK_DISPATCHER = "gearpump.task-dispatcher" val GEARPUMP_CLUSTER_MASTERS = "gearpump.cluster.masters" val GEARPUMP_MASTERCLIENT_TIMEOUT = "gearpump.masterclient.timeout" - val GEARPUMP_CLUSTER_EXECUTOR_WORKER_SHARE_SAME_PROCESS = "gearpump.worker.executor-share-same-jvm-as-worker" + val GEARPUMP_CLUSTER_EXECUTOR_WORKER_SHARE_SAME_PROCESS = + "gearpump.worker.executor-share-same-jvm-as-worker" val GEARPUMP_HOME = "gearpump.home" val GEARPUMP_FULL_SCALA_VERSION = "gearpump.binary-version-with-scala-version" @@ -57,23 +58,24 @@ object Constants { val GEARPUMP_LOG_APPLICATION_DIR = "gearpump.log.application.dir" val HADOOP_CONF = "hadoopConf" - // Id used to identity Master JVM process in low level resource manager like YARN. // In YARN, it means the container Id. - val GEARPUMP_MASTER_RESOURCE_MANAGER_CONTAINER_ID = "gearpump.master-resource-manager-container-id" + val GEARPUMP_MASTER_RESOURCE_MANAGER_CONTAINER_ID = + "gearpump.master-resource-manager-container-id" // Id used to identity Worker JVM process in low level resource manager like YARN. // In YARN, it means the container Id. - val GEARPUMP_WORKER_RESOURCE_MANAGER_CONTAINER_ID = "gearpump.worker-resource-manager-container-id" + val GEARPUMP_WORKER_RESOURCE_MANAGER_CONTAINER_ID = + "gearpump.worker-resource-manager-container-id" // true or false val GEARPUMP_REMOTE_DEBUG_EXECUTOR_JVM = "gearpump.remote-debug-executor-jvm" val GEARPUMP_REMOTE_DEBUG_PORT = "gearpump.remote-debug-port" - // whether turn on GC log, true or false + // Whether to turn on GC log, true or false val GEARPUMP_VERBOSE_GC = "gearpump.verbose-gc" - // the time out for Future, like ask. + // The time out for Future, like ask. // !Important! This global timeout setting will also impact the UI // responsive time if set to too big. Please make sure you have // enough justification to change this global setting, otherwise @@ -101,15 +103,14 @@ object Constants { val GEARPUMP_APP_JAR = "gearpump.app.jar" val GEARPUMP_APP_NAME_PREFIX = "gearpump.app.name.prefix" - // where the jar is stored at. It can be a HDFS, or a local disk. + // Where the jar is stored at. It can be a HDFS, or a local disk. val GEARPUMP_APP_JAR_STORE_ROOT_PATH = "gearpump.jarstore.rootpath" - // Use java property -Dgearpump.config.file=xxx.conf to set customized configuration + // Uses java property -Dgearpump.config.file=xxx.conf to set customized configuration // Otherwise application.conf in classpath will be loaded val GEARPUMP_CUSTOM_CONFIG_FILE = "gearpump.config.file" - - //Metrics related + // Metrics related val GEARPUMP_METRIC_ENABLED = "gearpump.metrics.enabled" val GEARPUMP_METRIC_SAMPLE_RATE = "gearpump.metrics.sample-rate" val GEARPUMP_METRIC_REPORT_INTERVAL = "gearpump.metrics.report-interval-ms" @@ -117,13 +118,13 @@ object Constants { val GEARPUMP_METRIC_GRAPHITE_PORT = "gearpump.metrics.graphite.port" val GEARPUMP_METRIC_REPORTER = "gearpump.metrics.reporter" - // we will retain at max @RETAIN_HISTORY_HOURS history data + // Retains at max @RETAIN_HISTORY_HOURS history data val GEARPUMP_METRIC_RETAIN_HISTORY_DATA_HOURS = "gearpump.metrics.retainHistoryData.hours" - // time interval between two history data points. + // Time interval between two history data points. val GEARPUMP_RETAIN_HISTORY_DATA_INTERVAL_MS = "gearpump.metrics.retainHistoryData.intervalMs" - // we will retain at max @RETAIN_LATEST_SECONDS recent data points + // Retains at max @RETAIN_LATEST_SECONDS recent data points val GEARPUMP_RETAIN_RECENT_DATA_SECONDS = "gearpump.metrics.retainRecentData.seconds" // time interval between two recent data points. @@ -133,13 +134,13 @@ object Constants { // and shutdown itself val GEARPUMP_RESOURCE_ALLOCATION_TIMEOUT = "gearpump.resource-allocation-timeout-seconds" - //Service related + // Service related val GEARPUMP_SERVICE_HTTP = "gearpump.services.http" val GEARPUMP_SERVICE_HOST = "gearpump.services.host" val GEARPUMP_SERVICE_SUPERVISOR_PATH = "gearpump.services.supervisor-actor-path" val GEARPUMP_SERVICE_RENDER_CONFIG_CONCISE = "gearpump.services.config-render-option-concise" - //The partitioners provided by Gearpump + // The partitioners provided by Gearpump val BUILTIN_PARTITIONERS = Array( classOf[BroadcastPartitioner], classOf[CoLocationPartitioner], @@ -147,11 +148,10 @@ object Constants { classOf[ShuffleGroupingPartitioner], classOf[ShufflePartitioner]) - //Security related + // Security related val GEARPUMP_KEYTAB_FILE = "gearpump.keytab.file" val GEARPUMP_KERBEROS_PRINCIPAL = "gearpump.kerberos.principal" - val GEARPUMP_METRICS_MAX_LIMIT = "gearpump.metrics.akka.max-limit-on-query" val GEARPUMP_METRICS_AGGREGATORS = "gearpump.metrics.akka.metrics-aggregator-class" diff --git a/core/src/main/scala/io/gearpump/util/FileUtils.scala b/core/src/main/scala/io/gearpump/util/FileUtils.scala index b0e8446cd..15615874f 100644 --- a/core/src/main/scala/io/gearpump/util/FileUtils.scala +++ b/core/src/main/scala/io/gearpump/util/FileUtils.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,9 +18,10 @@ package io.gearpump.util +import java.io.{File, IOException} import java.nio.charset.Charset + import io.gearpump.google.common.io.Files -import java.io.{IOException, File} object FileUtils { private val UTF8 = Charset.forName("UTF-8") @@ -41,7 +42,7 @@ object FileUtils { Files.toByteArray(file) } - // recursively making all parent directories including itself + /** recursively making all parent directories including itself */ def forceMkdir(directory: File): Unit = { if (directory.exists() && directory.isFile) { throw new IOException(s"Failed to create directory ${directory.toString}, it already exist") diff --git a/core/src/main/scala/io/gearpump/util/Graph.scala b/core/src/main/scala/io/gearpump/util/Graph.scala index 6bff9dabe..8c343292f 100644 --- a/core/src/main/scala/io/gearpump/util/Graph.scala +++ b/core/src/main/scala/io/gearpump/util/Graph.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,14 +17,14 @@ */ package io.gearpump.util +import scala.annotation.tailrec import scala.collection.mutable import scala.language.implicitConversions /** * Generic mutable Graph libraries. - * */ -class Graph[N, E](vertexList: List[N], edgeList: List[(N, E, N)]) extends Serializable{ +class Graph[N, E](vertexList: List[N], edgeList: List[(N, E, N)]) extends Serializable { private val _vertices = mutable.Set.empty[N] private val _edges = mutable.Set.empty[(N, E, N)] @@ -50,7 +50,7 @@ class Graph[N, E](vertexList: List[N], edgeList: List[(N, E, N)]) extends Serial * Add a vertex * Current Graph is changed. */ - def addVertex(vertex : N): Unit = { + def addVertex(vertex: N): Unit = { val result = _vertices.add(vertex) if (result) { _indexs += vertex -> nextId @@ -73,14 +73,14 @@ class Graph[N, E](vertexList: List[N], edgeList: List[(N, E, N)]) extends Serial * The result is stable */ def vertices: List[N] = { - // sort the vertex so that we can keep the order for mapVertex + // Sorts the vertex so that we can keep the order for mapVertex _vertices.toList.sortBy(_indexs(_)) } /** * out degree */ - def outDegreeOf(node : N): Int = { + def outDegreeOf(node: N): Int = { edges.count(_._1 == node) } @@ -94,7 +94,7 @@ class Graph[N, E](vertexList: List[N], edgeList: List[(N, E, N)]) extends Serial /** * out going edges. */ - def outgoingEdgesOf(node : N): List[(N, E, N)] = { + def outgoingEdgesOf(node: N): List[(N, E, N)] = { edges.filter(_._1 == node) } @@ -129,7 +129,7 @@ class Graph[N, E](vertexList: List[N], edgeList: List[(N, E, N)]) extends Serial * add edge * Current Graph is changed. */ - def addEdge(node1 : N, edge: E, node2: N): Unit = { + def addEdge(node1: N, edge: E, node2: N): Unit = { addVertex(node1) addVertex(node2) addEdge((node1, edge, node2)) @@ -155,7 +155,7 @@ class Graph[N, E](vertexList: List[N], edgeList: List[(N, E, N)]) extends Serial * Current graph is not changed. */ def mapEdge[NewEdge](fun: (N, E, N) => NewEdge): Graph[N, NewEdge] = { - val newEdges = edges.map {edge => + val newEdges = edges.map { edge => (edge._1, fun(edge._1, edge._2, edge._3), edge._3) } new Graph(vertices, newEdges) @@ -164,7 +164,7 @@ class Graph[N, E](vertexList: List[N], edgeList: List[(N, E, N)]) extends Serial /** * edges connected to node */ - def edgesOf(node : N): List[(N, E, N)] = { + def edgesOf(node: N): List[(N, E, N)] = { (incomingEdgesOf(node) ++ outgoingEdgesOf(node)).toSet[(N, E, N)].toList.sortBy(_indexs(_)) } @@ -179,9 +179,9 @@ class Graph[N, E](vertexList: List[N], edgeList: List[(N, E, N)]) extends Serial * Add another graph * Current graph is changed. */ - def addGraph(other : Graph[N, E]) : Graph[N, E] = { + def addGraph(other: Graph[N, E]): Graph[N, E] = { (vertices ++ other.vertices).foreach(addVertex(_)) - (edges ++ other.edges).foreach(edge =>addEdge(edge._1, edge._2, edge._3)) + (edges ++ other.edges).foreach(edge => addEdge(edge._1, edge._2, edge._3)) this } @@ -247,7 +247,7 @@ class Graph[N, E](vertexList: List[N], edgeList: List[(N, E, N)]) extends Serial val newGraph = copy var output = List.empty[N] - while(!newGraph.isEmpty) { + while (!newGraph.isEmpty) { output ++= newGraph.removeZeroInDegree } output.iterator @@ -255,6 +255,7 @@ class Graph[N, E](vertexList: List[N], edgeList: List[(N, E, N)]) extends Serial /** * Return all circles in graph. + * * The reference of this algorithm is: * https://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm */ @@ -278,11 +279,13 @@ class Graph[N, E](vertexList: List[N], edgeList: List[(N, E, N)]) extends Serial edge => { if (!indexMap.contains(edge._3)) { tarjan(edge._3) - if (lowLink.get(edge._3).get < lowLink.get(node).get) + if (lowLink.get(edge._3).get < lowLink.get(node).get) { lowLink(node) = lowLink(edge._3) + } } else { - if (inStack.get(edge._3).get && (indexMap.get(edge._3).get < lowLink.get(node).get)) + if (inStack.get(edge._3).get && (indexMap.get(edge._3).get < lowLink.get(node).get)) { lowLink(node) = indexMap(edge._3) + } } } } @@ -311,6 +314,7 @@ class Graph[N, E](vertexList: List[N], edgeList: List[(N, E, N)]) extends Serial /** * Return an iterator of vertex in topological order of graph with circles * The node returned by Iterator is stable sorted. + * * The reference of this algorithm is: * http://www.drdobbs.com/database/topological-sorting/184410262 */ @@ -345,10 +349,11 @@ class Graph[N, E](vertexList: List[N], edgeList: List[(N, E, N)]) extends Serial * check whether there is a loop */ def hasCycle(): Boolean = { - @annotation.tailrec def detectCycle(graph: Graph[N, E]): Boolean = { - if(graph.edges.isEmpty) { + @tailrec + def detectCycle(graph: Graph[N, E]): Boolean = { + if (graph.edges.isEmpty) { false - } else if(graph.vertices.nonEmpty && !graph.vertices.exists(graph.inDegreeOf(_) == 0)) { + } else if (graph.vertices.nonEmpty && !graph.vertices.exists(graph.inDegreeOf(_) == 0)) { true } else { graph.removeZeroInDegree @@ -367,23 +372,25 @@ class Graph[N, E](vertexList: List[N], edgeList: List[(N, E, N)]) extends Serial } /** - * Generate a level map for each vertex - * withholding: if vertex A -> B, then level(A) < level(B) + * Generate a level map for each vertex withholding: + * {{{ + * if vertex A -> B, then level(A) -> level(B) + * }}} */ def vertexHierarchyLevelMap(): Map[N, Int] = { val newGraph = copy var output = Map.empty[N, Int] var level = 0 - while(!newGraph.isEmpty) { + while (!newGraph.isEmpty) { output ++= newGraph.removeZeroInDegree.map((_, level)).toMap level += 1 } output } - override def toString = { + override def toString: String = { Map("vertices" -> vertices.mkString(","), - "edges" -> edges.mkString(",")).toString() + "edges" -> edges.mkString(",")).toString() } } @@ -391,6 +398,8 @@ object Graph { /** * Example: + * + * {{{ * Graph(1 ~ 2 ~> 4 ~ 5 ~> 7, 8~9~>55, 11) * Will create a graph with: * nodes: @@ -399,17 +408,17 @@ object Graph { * 2: (1->4) * 5: (4->7) * 9: (8->55) - * + * }}} */ def apply[N, E](elems: Path[_ <: N, _ <: E]*): Graph[N, E] = { val graph = empty[N, E] - elems.foreach{ path => + elems.foreach { path => path.updategraph(graph) } graph } - def apply[N , E](vertices: List[N], edges: List[(N, E, N)]): Graph[N, E] = { + def apply[N, E](vertices: List[N], edges: List[(N, E, N)]): Graph[N, E] = { new Graph(vertices, edges) } @@ -417,11 +426,11 @@ object Graph { Some((graph.vertices, graph.edges)) } - def empty[N, E] = { + def empty[N, E]: Graph[N, E] = { new Graph(List.empty[N], List.empty[(N, E, N)]) } - class Path[N, +E](path: List[Either[N, E]]) { + class Path[N, + E](path: List[Either[N, E]]) { def ~[Edge >: E](edge: Edge): Path[N, Edge] = { new Path(path :+ Right(edge)) @@ -458,7 +467,7 @@ object Graph { } implicit class Node[N, E](self: N) extends Path[N, E](List(Left(self))) { - + override def ~[Edge](edge: Edge): Path[N, Edge] = { new Path(List(Left(self), Right(edge))) } diff --git a/core/src/main/scala/io/gearpump/util/HistoryMetricsService.scala b/core/src/main/scala/io/gearpump/util/HistoryMetricsService.scala index 20a3fe6cd..7552444c8 100644 --- a/core/src/main/scala/io/gearpump/util/HistoryMetricsService.scala +++ b/core/src/main/scala/io/gearpump/util/HistoryMetricsService.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,19 +19,19 @@ package io.gearpump.util import java.util +import scala.collection.mutable.ListBuffer import akka.actor.Actor import com.typesafe.config.Config +import org.slf4j.Logger + import io.gearpump.TimeStamp -import io.gearpump.cluster.ClientToMaster.{ReadOption, QueryHistoryMetrics} +import io.gearpump.cluster.ClientToMaster.{QueryHistoryMetrics, ReadOption} import io.gearpump.cluster.MasterToClient.{HistoryMetrics, HistoryMetricsItem} import io.gearpump.metrics.Metrics._ import io.gearpump.metrics.MetricsAggregator import io.gearpump.util.Constants._ -import io.gearpump.util.HistoryMetricsService.{HistoryMetricsStore, SkipAllAggregator, DummyMetricsAggregator, MetricsStore, HistoryMetricsConfig} -import org.slf4j.Logger - -import scala.collection.mutable.ListBuffer +import io.gearpump.util.HistoryMetricsService.{DummyMetricsAggregator, HistoryMetricsConfig, HistoryMetricsStore, SkipAllAggregator} /** * @@ -44,7 +44,6 @@ import scala.collection.mutable.ListBuffer * for each hour. * * For fine-grained data in last 5 min, there will be 1 sample point per 15 seconds. - * */ class HistoryMetricsService(name: String, config: HistoryMetricsConfig) extends Actor { private val LOG: Logger = LogUtil.getLogger(getClass, name = name) @@ -75,14 +74,15 @@ class HistoryMetricsService(name: String, config: HistoryMetricsConfig) extends } + ".*$" } - private def fetchMetricsHistory(pathPattern: String, readOption: ReadOption.ReadOption): List[HistoryMetricsItem] = { + private def fetchMetricsHistory(pathPattern: String, readOption: ReadOption.ReadOption) + : List[HistoryMetricsItem] = { val result = new ListBuffer[HistoryMetricsItem] val regex = toRegularExpression(pathPattern).r.pattern val iter = metricsStore.iterator - while(iter.hasNext) { + while (iter.hasNext) { val (name, store) = iter.next() val matcher = regex.matcher(name) @@ -95,7 +95,7 @@ class HistoryMetricsService(name: String, config: HistoryMetricsConfig) extends case ReadOption.ReadHistory => result.append(store.readHistory: _*) case _ => - //skip all other options. + // Skip all other options. } } } @@ -106,43 +106,42 @@ class HistoryMetricsService(name: String, config: HistoryMetricsConfig) extends private var aggregators: Map[String, MetricsAggregator] = Map.empty[String, MetricsAggregator] import scala.collection.JavaConverters._ - val validAggregators = { - systemConfig.getConfig(Constants.GEARPUMP_METRICS_AGGREGATORS).root.unwrapped.keySet().asScala.toSet + private val validAggregators: Set[String] = { + val rootConfig = systemConfig.getConfig(Constants.GEARPUMP_METRICS_AGGREGATORS).root.unwrapped + rootConfig.keySet().asScala.toSet } def commandHandler: Receive = { - //path accept syntax ? *, ? will match one char, * will match at least one char + // Path accept syntax ? *, ? will match one char, * will match at least one char case QueryHistoryMetrics(inputPath, readOption, aggregatorClazz, options) => - val aggregator = { - if (aggregatorClazz == null || aggregatorClazz.isEmpty) { - dummyAggregator - } else if (aggregators.contains(aggregatorClazz)) { - aggregators(aggregatorClazz) - } else if (validAggregators.contains(aggregatorClazz)) { - val clazz = Class.forName(aggregatorClazz) - val constructor = clazz.getConstructor(classOf[Config]) - val aggregator = constructor.newInstance(systemConfig).asInstanceOf[MetricsAggregator] - aggregators += aggregatorClazz -> aggregator - aggregator - } else { - LOG.error(s"Aggregator $aggregatorClazz is not in the white list ${validAggregators}, we will drop all messages. " + - s"Please see config at ${GEARPUMP_METRICS_AGGREGATORS}") - val skipAll = new SkipAllAggregator - aggregators += aggregatorClazz -> new SkipAllAggregator - skipAll - } - } - - import collection.JavaConversions._ - val metrics = fetchMetricsHistory(inputPath, readOption).iterator + val aggregator = { + if (aggregatorClazz == null || aggregatorClazz.isEmpty) { + dummyAggregator + } else if (aggregators.contains(aggregatorClazz)) { + aggregators(aggregatorClazz) + } else if (validAggregators.contains(aggregatorClazz)) { + val clazz = Class.forName(aggregatorClazz) + val constructor = clazz.getConstructor(classOf[Config]) + val aggregator = constructor.newInstance(systemConfig).asInstanceOf[MetricsAggregator] + aggregators += aggregatorClazz -> aggregator + aggregator + } else { + LOG.error(s"Aggregator $aggregatorClazz is not in the white list ${validAggregators}, " + + s"we will drop all messages. Please see config at ${GEARPUMP_METRICS_AGGREGATORS}") + val skipAll = new SkipAllAggregator + aggregators += aggregatorClazz -> new SkipAllAggregator + skipAll + } + } + + val metrics = fetchMetricsHistory(inputPath, readOption).iterator sender ! HistoryMetrics(inputPath, aggregator.aggregate(options, metrics)) - } + } } object HistoryMetricsService { - trait MetricsStore { def add(inputMetrics: MetricType): Unit @@ -169,7 +168,7 @@ object HistoryMetricsService { def readHistory: List[HistoryMetricsItem] } - class DummyHistoryMetricsStore extends HistoryMetricsStore{ + class DummyHistoryMetricsStore extends HistoryMetricsStore { val empty = List.empty[HistoryMetricsItem] @@ -187,7 +186,8 @@ object HistoryMetricsService { } object HistoryMetricsStore { - def apply(name: String, metric: MetricType, config: HistoryMetricsConfig): HistoryMetricsStore = { + def apply(name: String, metric: MetricType, config: HistoryMetricsConfig) + : HistoryMetricsStore = { metric match { case histogram: Histogram => new HistogramMetricsStore(config) case meter: Meter => new MeterMetricsStore(config) @@ -199,18 +199,18 @@ object HistoryMetricsService { } /** - ** Metrics store to store history data points + * Metrics store to store history data points * For each time point, we will store single data point. * * @param retainCount how many data points to retain, old data will be removed * @param retainIntervalMs time interval between two data points. */ - class SingleValueMetricsStore (retainCount: Int, retainIntervalMs: Long) extends MetricsStore{ + class SingleValueMetricsStore(retainCount: Int, retainIntervalMs: Long) extends MetricsStore { - private val queue = new util.ArrayDeque[HistoryMetricsItem]() + private val queue = new util.ArrayDeque[HistoryMetricsItem]() private var latest = List.empty[HistoryMetricsItem] - // end of the time window we are tracking + // End of the time window we are tracking private var endTime = 0L override def add(inputMetrics: MetricType): Unit = { @@ -226,7 +226,7 @@ object HistoryMetricsService { queue.addFirst(metrics) endTime = (now / retainIntervalMs + 1) * retainIntervalMs - // remove old data + // Removes old data if (queue.size() > retainCount) { queue.removeLast() } @@ -235,8 +235,8 @@ object HistoryMetricsService { def read: List[HistoryMetricsItem] = { val result = new ListBuffer[HistoryMetricsItem] - import scala.collection.JavaConversions.asScalaIterator - queue.iterator().foreach(result.prepend(_)) + import scala.collection.JavaConverters._ + queue.iterator().asScala.foreach(result.prepend(_)) result.toList } @@ -246,11 +246,14 @@ object HistoryMetricsService { } /** + * Config for how long to keep history metrics data. * * @param retainHistoryDataHours Retain at max @RETAIN_HISTORY_HOURS history data(unit hour) * @param retainHistoryDataIntervalMs time interval between two history data points.(unit: ms) - * @param retainRecentDataSeconds Retain at max @RETAIN_LATEST_SECONDS recent data points(unit: seconds) - * @param retainRecentDataIntervalMs Retain at max @RETAIN_LATEST_SECONDS recent data points(unit: ms) + * @param retainRecentDataSeconds Retain at max @RETAIN_LATEST_SECONDS + * recent data points(unit: seconds) + * @param retainRecentDataIntervalMs Retain at max @RETAIN_LATEST_SECONDS recent + * data points(unit: ms) */ case class HistoryMetricsConfig( retainHistoryDataHours: Int, @@ -385,14 +388,17 @@ object HistoryMetricsService { } class DummyMetricsAggregator extends MetricsAggregator { - def aggregate(options: Map[String, String], inputs: Iterator[HistoryMetricsItem]): List[HistoryMetricsItem] = { - import scala.collection.JavaConverters._ + def aggregate(options: Map[String, String], inputs: Iterator[HistoryMetricsItem]) + : List[HistoryMetricsItem] = { inputs.toList } } class SkipAllAggregator extends MetricsAggregator { private val empty = List.empty[HistoryMetricsItem] - def aggregate(options: Map[String, String], inputs: Iterator[HistoryMetricsItem]): List[HistoryMetricsItem] = empty + def aggregate(options: Map[String, String], inputs: Iterator[HistoryMetricsItem]) + : List[HistoryMetricsItem] = { + empty + } } } diff --git a/core/src/main/scala/io/gearpump/util/LogUtil.scala b/core/src/main/scala/io/gearpump/util/LogUtil.scala index 5fa3136d1..166912983 100644 --- a/core/src/main/scala/io/gearpump/util/LogUtil.scala +++ b/core/src/main/scala/io/gearpump/util/LogUtil.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,27 +16,26 @@ * limitations under the License. */ - package io.gearpump.util import java.io.File import java.net.InetAddress import java.util.Properties +import scala.util.Try import com.typesafe.config.Config import org.apache.log4j.PropertyConfigurator import org.slf4j.{Logger, LoggerFactory} -import scala.util.Try - object LogUtil { object ProcessType extends Enumeration { type ProcessType = Value val MASTER, WORKER, LOCAL, APPLICATION, UI = Value } - def getLogger[T](clazz : Class[T], context : String = null, master : Any = null, worker : Any = null, executor : Any = null, task : Any = null, app : Any = null, name: String = null) : Logger = { - + def getLogger[T]( + clazz: Class[T], context: String = null, master: Any = null, worker: Any = null, + executor: Any = null, task: Any = null, app: Any = null, name: String = null): Logger = { var env = "" if (null != context) { @@ -70,8 +69,9 @@ object LogUtil { } } - def loadConfiguration(config : Config, processType : ProcessType.ProcessType) : Unit = { - //set log file name + /** Custom the log file locations by reading config from system properties */ + def loadConfiguration(config: Config, processType: ProcessType.ProcessType): Unit = { + // Set log file name val propName = s"gearpump.${processType.toString.toLowerCase}.log.file" val props = loadConfiguration @@ -82,7 +82,8 @@ object LogUtil { processType match { case ProcessType.APPLICATION => props.setProperty("log4j.rootAppender", "${gearpump.application.logger}") - props.setProperty("gearpump.application.log.rootdir", applicationLogDir(config).getAbsolutePath) + props.setProperty("gearpump.application.log.rootdir", + applicationLogDir(config).getAbsolutePath) case _ => props.setProperty("log4j.rootAppender", "${gearpump.root.logger}") props.setProperty("gearpump.log.dir", daemonLogDir(config).getAbsolutePath) @@ -96,7 +97,7 @@ object LogUtil { new File(dir) } - def verboseLogToConsole: Unit = { + def verboseLogToConsole(): Unit = { val props = loadConfiguration props.setProperty("log4j.rootLogger", "DEBUG,console") PropertyConfigurator.configure(props) @@ -105,14 +106,14 @@ object LogUtil { def loadConfiguration: Properties = { val props = new Properties() val log4jConfStream = getClass().getClassLoader.getResourceAsStream("log4j.properties") - if(log4jConfStream!=null) { + if (log4jConfStream != null) { props.load(log4jConfStream) } log4jConfStream.close() props } - private def jvmName : String = { + private def jvmName: String = { val hostname = Try(InetAddress.getLocalHost.getHostName).getOrElse("local") java.lang.management.ManagementFactory.getRuntimeMXBean().getName() } diff --git a/core/src/main/scala/io/gearpump/util/ProcessLogRedirector.scala b/core/src/main/scala/io/gearpump/util/ProcessLogRedirector.scala index 13f9ea8fa..0b843f3db 100644 --- a/core/src/main/scala/io/gearpump/util/ProcessLogRedirector.scala +++ b/core/src/main/scala/io/gearpump/util/ProcessLogRedirector.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,11 +19,11 @@ package io.gearpump.util import java.io.{Closeable, Flushable} +import scala.sys.process.ProcessLogger import org.slf4j.LoggerFactory -import scala.sys.process.ProcessLogger - +/** Redirect the console output to parent process */ class ProcessLogRedirector extends ProcessLogger with Closeable with Flushable with ConsoleOutput { private val LOG = LoggerFactory.getLogger("redirect") diff --git a/core/src/main/scala/io/gearpump/util/ReferenceEqual.scala b/core/src/main/scala/io/gearpump/util/ReferenceEqual.scala index 1142bdcdb..f6c7a2bcd 100644 --- a/core/src/main/scala/io/gearpump/util/ReferenceEqual.scala +++ b/core/src/main/scala/io/gearpump/util/ReferenceEqual.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,13 +18,16 @@ package io.gearpump.util - /** * Check equal using reference-equal. */ trait ReferenceEqual extends AnyRef { - override def equals(other : Any) : Boolean = { + override def equals(other: Any): Boolean = { this.eq(other.asInstanceOf[AnyRef]) } + + override def hashCode(): Int = { + super.hashCode() + } } diff --git a/core/src/main/scala/io/gearpump/util/RestartPolicy.scala b/core/src/main/scala/io/gearpump/util/RestartPolicy.scala index 787b508c0..245cb1ba4 100644 --- a/core/src/main/scala/io/gearpump/util/RestartPolicy.scala +++ b/core/src/main/scala/io/gearpump/util/RestartPolicy.scala @@ -15,18 +15,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.gearpump.util -import akka.actor.{ChildRestartStats, ActorRef} +package io.gearpump.util import scala.concurrent.duration.Duration +import akka.actor.ChildRestartStats + /** - * @param maxNrOfRetries the number of times is allowed to be restarted, negative value means no limit, - * if the limit is exceeded the policy will not allow to restart - * @param withinTimeRange duration of the time window for maxNrOfRetries, Duration.Inf means no window + * When one executor or task fails, Gearpump will try to start. However, if it fails after + * multiple retries, then we abort. + * + * @param maxNrOfRetries The number of times is allowed to be restarted, negative value means no + * limit, if the limit is exceeded the policy will not allow to restart + * @param withinTimeRange Duration of the time window for maxNrOfRetries. + * Duration.Inf means no window */ -class RestartPolicy (maxNrOfRetries: Int, withinTimeRange: Duration) { +class RestartPolicy(maxNrOfRetries: Int, withinTimeRange: Duration) { private val status = new ChildRestartStats(null, 0, 0L) private val retriesWindow = (Some(maxNrOfRetries), Some(withinTimeRange.toMillis.toInt)) diff --git a/core/src/main/scala/io/gearpump/util/RichProcess.scala b/core/src/main/scala/io/gearpump/util/RichProcess.scala index bfe060713..ab5611fd9 100644 --- a/core/src/main/scala/io/gearpump/util/RichProcess.scala +++ b/core/src/main/scala/io/gearpump/util/RichProcess.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -25,7 +25,9 @@ trait ConsoleOutput { def error: String } -class RichProcess(process: Process, val logger: ConsoleOutput) extends Process { - def exitValue() : scala.Int = process.exitValue() - def destroy() : scala.Unit = process.destroy() +/** Extends Process by providing a additional logger: ConsoleOutput interface. */ +class RichProcess(process: Process, _logger: ConsoleOutput) extends Process { + def exitValue(): scala.Int = process.exitValue() + def destroy(): scala.Unit = process.destroy() + def logger: ConsoleOutput = _logger } diff --git a/core/src/main/scala/io/gearpump/util/TimeOutScheduler.scala b/core/src/main/scala/io/gearpump/util/TimeOutScheduler.scala index 1464b2174..64b920cde 100644 --- a/core/src/main/scala/io/gearpump/util/TimeOutScheduler.scala +++ b/core/src/main/scala/io/gearpump/util/TimeOutScheduler.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,20 +15,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package io.gearpump.util import java.util.concurrent.TimeUnit +import scala.concurrent.duration._ import akka.actor.{Actor, ActorRef} import akka.pattern.ask -import scala.concurrent.duration._ - +/** A helper util to send a message to remote actor and notify callback when timeout */ trait TimeOutScheduler { this: Actor => import context.dispatcher - def sendMsgWithTimeOutCallBack(target: ActorRef, msg: AnyRef, milliSeconds: Long, timeOutHandler: => Unit): Unit = { + def sendMsgWithTimeOutCallBack( + target: ActorRef, msg: AnyRef, milliSeconds: Long, timeOutHandler: => Unit): Unit = { val result = target.ask(msg)(FiniteDuration(milliSeconds, TimeUnit.MILLISECONDS)) result onSuccess { case msg => diff --git a/core/src/main/scala/io/gearpump/util/Util.scala b/core/src/main/scala/io/gearpump/util/Util.scala index 96eddfdf8..8ed9bb34d 100644 --- a/core/src/main/scala/io/gearpump/util/Util.scala +++ b/core/src/main/scala/io/gearpump/util/Util.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,18 +18,18 @@ package io.gearpump.util -import java.io.{DataInputStream, FileInputStream, File} -import java.net.{URL, URI, ServerSocket} -import java.util.jar.Manifest -import com.typesafe.config.{ConfigFactory, Config} -import io.gearpump.cluster.AppJar -import io.gearpump.jarstore.{FilePath, JarStoreService} -import io.gearpump.transport.HostPort - +import java.io.{BufferedReader, File, FileInputStream, InputStreamReader} +import java.net.{ServerSocket, URI} import scala.concurrent.forkjoin.ThreadLocalRandom -import scala.sys.process.{ProcessLogger, Process} +import scala.sys.process.Process import scala.util.{Failure, Success, Try} +import com.typesafe.config.{Config, ConfigFactory} + +import io.gearpump.cluster.AppJar +import io.gearpump.jarstore.JarStoreService +import io.gearpump.transport.HostPort + object Util { val LOG = LogUtil.getLogger(getClass) private val defaultUri = new URI("file:///") @@ -39,7 +39,7 @@ object Util { appNamePattern.matcher(appName).matches() } - def getCurrentClassPath : Array[String] = { + def getCurrentClassPath: Array[String] = { val classpath = System.getProperty("java.class.path") val classpathList = classpath.split(File.pathSeparator) classpathList @@ -48,8 +48,9 @@ object Util { def version: String = { val home = System.getProperty(Constants.GEARPUMP_HOME) val version = Try { - val versionFile = new DataInputStream(new FileInputStream(new File(home, "VERSION"))) - val version = versionFile.readLine().replace("version:=", "") + val versionFile = new FileInputStream(new File(home, "VERSION")) + val reader = new BufferedReader(new InputStreamReader(versionFile)) + val version = reader.readLine().replace("version:=", "") versionFile.close() version } @@ -62,11 +63,13 @@ object Util { } } - def startProcess(options : Array[String], classPath : Array[String], mainClass : String, - arguments : Array[String]) : RichProcess = { + def startProcess(options: Array[String], classPath: Array[String], mainClass: String, + arguments: Array[String]): RichProcess = { val java = System.getProperty("java.home") + "/bin/java" - val command = List(java) ++ options ++ List("-cp", classPath.mkString(File.pathSeparator), mainClass) ++ arguments - LOG.info(s"Starting executor process java $mainClass ${arguments.mkString(" ")} \n ${options.mkString(" ")}") + val command = List(java) ++ options ++ + List("-cp", classPath.mkString(File.pathSeparator), mainClass) ++ arguments + LOG.info(s"Starting executor process java $mainClass ${arguments.mkString(" ")} " + + s"\n ${options.mkString(" ")}") val logger = new ProcessLogRedirector() val process = Process(command).run(logger) new RichProcess(process, logger) @@ -75,7 +78,7 @@ object Util { /** * hostList format: host1:port1,host2:port2,host3:port3... */ - def parseHostList(hostList : String) : List[HostPort] = { + def parseHostList(hostList: String): List[HostPort] = { val masters = hostList.trim.split(",").map { address => val hostAndPort = address.split(":") HostPort(hostAndPort(0), hostAndPort(1).toInt) @@ -85,7 +88,7 @@ object Util { def resolvePath(path: String): String = { val uri = new URI(path) - if(uri.getScheme == null && uri.getFragment == null) { + if (uri.getScheme == null && uri.getFragment == null) { val absolutePath = new File(path).getCanonicalPath.replaceAll("\\\\", "/") "file://" + absolutePath } else { @@ -106,16 +109,16 @@ object Util { } } - def randInt: Int = { + def randInt(): Int = { Math.abs(ThreadLocalRandom.current.nextInt()) } - def findFreePort: Try[Int] = { + def findFreePort(): Try[Int] = { Try { - val socket = new ServerSocket(0); - socket.setReuseAddress(true); - val port = socket.getLocalPort(); - socket.close; + val socket = new ServerSocket(0) + socket.setReuseAddress(true) + val port = socket.getLocalPort() + socket.close port } } @@ -132,7 +135,6 @@ object Util { * Then you can use like this: * * filterOutOrigin(config, "reference.conf") - * */ import scala.collection.JavaConverters._ def filterOutOrigin(config: Config, originFile: String): Config = { @@ -148,16 +150,19 @@ object Util { } } - case class JvmSetting(vmargs : Array[String], classPath : Array[String]) + case class JvmSetting(vmargs: Array[String], classPath: Array[String]) - case class AppJvmSettings(appMater : JvmSetting, executor : JvmSetting) + case class AppJvmSettings(appMater: JvmSetting, executor: JvmSetting) - def resolveJvmSetting(conf : Config) : AppJvmSettings = { + /** Get an effective AppJvmSettings from Config */ + def resolveJvmSetting(conf: Config): AppJvmSettings = { - import Constants._ + import io.gearpump.util.Constants._ - val appMasterVMArgs = Try(conf.getString(GEARPUMP_APPMASTER_ARGS).split("\\s+").filter(_.nonEmpty)).toOption - val executorVMArgs = Try(conf.getString(GEARPUMP_EXECUTOR_ARGS).split("\\s+").filter(_.nonEmpty)).toOption + val appMasterVMArgs = Try(conf.getString(GEARPUMP_APPMASTER_ARGS).split("\\s+") + .filter(_.nonEmpty)).toOption + val executorVMArgs = Try(conf.getString(GEARPUMP_EXECUTOR_ARGS).split("\\s+") + .filter(_.nonEmpty)).toOption val appMasterClassPath = Try( conf.getString(GEARPUMP_APPMASTER_EXTRA_CLASSPATH) @@ -169,7 +174,7 @@ object Util { AppJvmSettings( JvmSetting(appMasterVMArgs.getOrElse(Array.empty[String]), - appMasterClassPath.getOrElse(Array.empty[String]) ), + appMasterClassPath.getOrElse(Array.empty[String])), JvmSetting(executorVMArgs .getOrElse(Array.empty[String]), executorClassPath.getOrElse(Array.empty[String]))) } diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties index 870a69f19..0faadd9df 100644 --- a/core/src/test/resources/log4j.properties +++ b/core/src/test/resources/log4j.properties @@ -7,7 +7,7 @@ # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -50,7 +50,6 @@ log4j.rootLogger=INFO,console # Logging Threshold log4j.threshhold=ALL - # ===================================================================== # Appenders # ===================================================================== @@ -115,4 +114,3 @@ log4j.appender.ALA.layout=org.apache.log4j.PatternLayout #log4j.appender.ALA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n log4j.appender.ALA.layout.ConversionPattern=[%p] [%d{MM/dd/yyyy HH:mm:ss.SSS}] [%c{1}] %m%n - diff --git a/core/src/test/resources/test.conf b/core/src/test/resources/test.conf index 324e8bd46..ac18c88e6 100644 --- a/core/src/test/resources/test.conf +++ b/core/src/test/resources/test.conf @@ -97,7 +97,6 @@ gearpump-ui { } } - akka { logger-startup-timeout = 30s @@ -111,7 +110,6 @@ akka { mailbox-type = "akka.dispatch.SingleConsumerOnlyUnboundedMailbox" } default-dispatcher { - mailbox-type = "akka.dispatch.SingleConsumerOnlyUnboundedMailbox" throughput = 10 fork-join-executor { parallelism-factor = 2 @@ -145,7 +143,6 @@ akka { default-remote-dispatcher { throughput = 5 type = Dispatcher - mailbox-type = "akka.dispatch.SingleConsumerOnlyUnboundedMailbox" executor = "fork-join-executor" fork-join-executor { parallelism-min = 1 diff --git a/core/src/test/scala/io/gearpump/TestProbeUtil.scala b/core/src/test/scala/io/gearpump/TestProbeUtil.scala index 9b808ddbb..e7181dba4 100644 --- a/core/src/test/scala/io/gearpump/TestProbeUtil.scala +++ b/core/src/test/scala/io/gearpump/TestProbeUtil.scala @@ -18,11 +18,13 @@ package io.gearpump -import akka.actor.{Terminated, Actor, Props} +import scala.language.implicitConversions + +import akka.actor.{Actor, Props, Terminated} import akka.testkit.TestProbe object TestProbeUtil { - implicit def toProps(probe: TestProbe) = { + implicit def toProps(probe: TestProbe): Props = { Props(new Actor { val probeRef = probe.ref context.watch(probeRef) diff --git a/core/src/test/scala/io/gearpump/cluster/MasterHarness.scala b/core/src/test/scala/io/gearpump/cluster/MasterHarness.scala index 21d145a66..b1ca3577d 100644 --- a/core/src/test/scala/io/gearpump/cluster/MasterHarness.scala +++ b/core/src/test/scala/io/gearpump/cluster/MasterHarness.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,24 +19,20 @@ package io.gearpump.cluster import java.io.File -import io.gearpump.util.{Constants, ActorUtil} -import io.gearpump.util.{FileUtils} -import java.net.{UnknownHostException, SocketTimeoutException, Socket, InetSocketAddress, ServerSocket, URLClassLoader} +import java.net.{InetSocketAddress, Socket, SocketTimeoutException, URLClassLoader, UnknownHostException} import java.util.Properties import java.util.concurrent.{Executors, TimeUnit} +import scala.collection.JavaConverters._ +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, ExecutionContext} import akka.actor.{Actor, ActorSystem, Address, Props} import akka.testkit.TestProbe import com.typesafe.config.{Config, ConfigFactory, ConfigParseOptions, ConfigValueFactory} + import io.gearpump.cluster.MasterHarness.MockMaster import io.gearpump.util.Constants._ -import io.gearpump.util.{Constants, LogUtil, ActorUtil, Util} - -import scala.collection.JavaConverters._ -import scala.concurrent.ExecutionContext -import scala.concurrent.duration.Duration -import scala.sys.process.Process -import scala.util.Try +import io.gearpump.util.{ActorUtil, FileUtils, LogUtil} trait MasterHarness { private val LOG = LogUtil.getLogger(getClass) @@ -54,7 +50,7 @@ trait MasterHarness { def getHost: String = host def getPort: Int = port - def config : Config + protected def config: Config def startActorSystem(): Unit = { val systemConfig = config @@ -69,13 +65,13 @@ trait MasterHarness { LOG.info(s"Actor system is started, $host, $port") } - def shutdownActorSystem():Unit = { - system.shutdown() - system.awaitTermination + def shutdownActorSystem(): Unit = { + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) LOG.info(s"Actor system is stopped, $host, $port") } - def convertTestConf(host : String, port : Int) : File = { + def convertTestConf(host: String, port: Int): File = { val test = ConfigFactory.parseResourcesAnySyntax("test.conf", ConfigParseOptions.defaults.setAllowMissing(true)) @@ -88,13 +84,13 @@ trait MasterHarness { confFile } - def createMockMaster() : TestProbe = { + def createMockMaster(): TestProbe = { val masterReceiver = TestProbe()(system) val master = system.actorOf(Props(classOf[MockMaster], masterReceiver), MASTER) masterReceiver } - def isPortUsed(host : String, port : Int) : Boolean = { + def isPortUsed(host: String, port: Int): Boolean = { var isPortUsed = true val socket = new Socket() @@ -103,12 +99,12 @@ trait MasterHarness { socket.connect(new InetSocketAddress(host, port), 1000) socket.isConnected } catch { - case ex: SocketTimeoutException => + case ex: SocketTimeoutException => isPortUsed = false case ex: UnknownHostException => isPortUsed = false case ex: Throwable => - // for other case, we think the port is listened + // For other case, we think the port has been occupied. isPortUsed = true } finally { socket.close() @@ -116,7 +112,7 @@ trait MasterHarness { isPortUsed } - def getContextClassPath : Array[String] = { + def getContextClassPath: Array[String] = { val contextLoader = Thread.currentThread().getContextClassLoader() val urlLoader = if (!contextLoader.isInstanceOf[URLClassLoader]) { @@ -135,19 +131,16 @@ trait MasterHarness { /** * Remove trailing $ */ - def getMainClassName(mainObj : Any) : String = { + def getMainClassName(mainObj: Any): String = { mainObj.getClass.getName.dropRight(1) } - import Constants._ - def getMasterListOption(): Array[String] = { masterProperties.asScala.toList.map { kv => s"-D${kv._1}=${kv._2}" }.toArray } - def masterConfig: Config = { ConfigFactory.parseProperties(masterProperties).withFallback(system.settings.config) } diff --git a/core/src/test/scala/io/gearpump/cluster/TestUtil.scala b/core/src/test/scala/io/gearpump/cluster/TestUtil.scala index e491fc77f..dce490239 100644 --- a/core/src/test/scala/io/gearpump/cluster/TestUtil.scala +++ b/core/src/test/scala/io/gearpump/cluster/TestUtil.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package io.gearpump.cluster import akka.actor._ @@ -25,10 +26,11 @@ object TestUtil { val UI_CONFIG = ClusterConfig.ui("test.conf") class DummyAppMaster(context: AppMasterContext, app: AppDescription) extends ApplicationMaster { - context.masterProxy ! (context, app) + context.masterProxy !(context, app) - def receive : Receive = null + def receive: Receive = null } - val dummyApp : AppDescription = AppDescription("dummy", classOf[DummyAppMaster].getName, UserConfig.empty) + val dummyApp: AppDescription = + AppDescription("dummy", classOf[DummyAppMaster].getName, UserConfig.empty) } \ No newline at end of file diff --git a/core/src/test/scala/io/gearpump/cluster/appmaster/AppMasterRuntimeEnvironmentSpec.scala b/core/src/test/scala/io/gearpump/cluster/appmaster/AppMasterRuntimeEnvironmentSpec.scala index 01d347403..00bd40885 100644 --- a/core/src/test/scala/io/gearpump/cluster/appmaster/AppMasterRuntimeEnvironmentSpec.scala +++ b/core/src/test/scala/io/gearpump/cluster/appmaster/AppMasterRuntimeEnvironmentSpec.scala @@ -18,8 +18,13 @@ package io.gearpump.cluster.appmaster +import scala.concurrent.Await +import scala.concurrent.duration.Duration + import akka.actor._ import akka.testkit.TestProbe +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} + import io.gearpump.TestProbeUtil._ import io.gearpump.cluster.AppMasterToMaster.RegisterAppMaster import io.gearpump.cluster._ @@ -27,27 +32,28 @@ import io.gearpump.cluster.appmaster.AppMasterRuntimeEnvironment._ import io.gearpump.cluster.appmaster.AppMasterRuntimeEnvironmentSpec.TestAppMasterEnv import io.gearpump.cluster.appmaster.ExecutorSystemScheduler.StartExecutorSystems import io.gearpump.cluster.appmaster.MasterConnectionKeeper.MasterConnectionStatus.{MasterConnected, MasterStopped} -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -class AppMasterRuntimeEnvironmentSpec extends FlatSpec with Matchers with BeforeAndAfterAll { +class AppMasterRuntimeEnvironmentSpec extends FlatSpec with Matchers with BeforeAndAfterAll { implicit var system: ActorSystem = null - override def beforeAll() = { + override def beforeAll(): Unit = { system = ActorSystem("test", TestUtil.DEFAULT_CONFIG) } - override def afterAll() = { - system.shutdown() - system.awaitTermination() + override def afterAll(): Unit = { + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } - "MasterWithExecutorSystemProvider" should "forward request StartExecutorSystem to ExecutorSystemProvider" in { + "MasterWithExecutorSystemProvider" should + "forward request StartExecutorSystem to ExecutorSystemProvider" in { val client = TestProbe() val master = TestProbe() val provider = TestProbe() val providerProps: Props = provider - val masterEnhanced = system.actorOf(Props(new MasterWithExecutorSystemProvider(master.ref, providerProps))) + val masterEnhanced = system.actorOf(Props( + new MasterWithExecutorSystemProvider(master.ref, providerProps))) val start = StartExecutorSystems(null, null) client.send(masterEnhanced, start) @@ -76,7 +82,8 @@ class AppMasterRuntimeEnvironmentSpec extends FlatSpec with Matchers with Before } "AppMasterRuntimeEnvironment" should "start appMaster when master is connected" in { - val TestAppMasterEnv(master, appMaster, masterConnectionKeeper, runtimeEnv) = setupAppMasterRuntimeEnv + val TestAppMasterEnv(master, appMaster, masterConnectionKeeper, runtimeEnv) = + setupAppMasterRuntimeEnv() masterConnectionKeeper.send(runtimeEnv, MasterConnected) appMaster.expectMsg(StartAppMaster) @@ -84,7 +91,8 @@ class AppMasterRuntimeEnvironmentSpec extends FlatSpec with Matchers with Before "AppMasterRuntimeEnvironment" should "shutdown itself when master is stopped" in { - val TestAppMasterEnv(master, appMaster, masterConnectionKeeper, runtimeEnv) = setupAppMasterRuntimeEnv + val TestAppMasterEnv(master, appMaster, masterConnectionKeeper, runtimeEnv) = + setupAppMasterRuntimeEnv() masterConnectionKeeper.send(runtimeEnv, MasterStopped) val client = TestProbe() @@ -94,7 +102,8 @@ class AppMasterRuntimeEnvironmentSpec extends FlatSpec with Matchers with Before "AppMasterRuntimeEnvironment" should "shutdown itself when appMaster is stopped" in { - val TestAppMasterEnv(master, appMaster, masterConnectionKeeper, runtimeEnv) = setupAppMasterRuntimeEnv + val TestAppMasterEnv(master, appMaster, masterConnectionKeeper, runtimeEnv) = + setupAppMasterRuntimeEnv() val client = TestProbe() client.watch(runtimeEnv) @@ -102,13 +111,13 @@ class AppMasterRuntimeEnvironmentSpec extends FlatSpec with Matchers with Before client.expectTerminated(runtimeEnv) } - private def setupAppMasterRuntimeEnv: TestAppMasterEnv = { - val appContext = AppMasterContext(0, null, null, null, null, null, null) + private def setupAppMasterRuntimeEnv(): TestAppMasterEnv = { + val appContext = AppMasterContext(0, null, null, null, null, null, null) val app = AppDescription("app", "AppMasterClass", null, null) val master = TestProbe() val masterFactory = (_: AppId, _: MasterActorRef) => toProps(master) val appMaster = TestProbe() - val appMasterFactory = (_: AppMasterContext, _: AppDescription)=> toProps(appMaster) + val appMasterFactory = (_: AppMasterContext, _: AppDescription) => toProps(appMaster) val masterConnectionKeeper = TestProbe() val masterConnectionKeeperFactory = (_: MasterActorRef, _: RegisterAppMaster, _: ListenerActorRef) => @@ -125,5 +134,7 @@ class AppMasterRuntimeEnvironmentSpec extends FlatSpec with Matchers with Before object AppMasterRuntimeEnvironmentSpec { - case class TestAppMasterEnv(master: TestProbe, appMaster: TestProbe, connectionkeeper: TestProbe, appMasterRuntimeEnv: ActorRef) + case class TestAppMasterEnv( + master: TestProbe, appMaster: TestProbe, connectionkeeper: TestProbe, + appMasterRuntimeEnv: ActorRef) } \ No newline at end of file diff --git a/core/src/test/scala/io/gearpump/cluster/appmaster/ExecutorSystemLauncherSpec.scala b/core/src/test/scala/io/gearpump/cluster/appmaster/ExecutorSystemLauncherSpec.scala index dc8e62455..359596036 100644 --- a/core/src/test/scala/io/gearpump/cluster/appmaster/ExecutorSystemLauncherSpec.scala +++ b/core/src/test/scala/io/gearpump/cluster/appmaster/ExecutorSystemLauncherSpec.scala @@ -18,24 +18,25 @@ package io.gearpump.cluster.appmaster +import scala.concurrent.Await +import scala.concurrent.duration._ + import akka.actor.{ActorSystem, Props} import akka.testkit.TestProbe import com.typesafe.config.ConfigValueFactory -import io.gearpump.WorkerId -import io.gearpump.cluster.AppMasterToWorker.LaunchExecutor +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} + import io.gearpump.cluster.AppMasterToWorker.LaunchExecutor import io.gearpump.cluster.TestUtil import io.gearpump.cluster.WorkerToAppMaster.ExecutorLaunchRejected import io.gearpump.cluster.appmaster.ExecutorSystemLauncher._ import io.gearpump.cluster.appmaster.ExecutorSystemScheduler.Session import io.gearpump.cluster.scheduler.Resource +import io.gearpump.cluster.worker.WorkerId import io.gearpump.util.ActorSystemBooter.{ActorSystemRegistered, RegisterActorSystem} import io.gearpump.util.Constants -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} - -import scala.concurrent.duration._ -class ExecutorSystemLauncherSpec extends FlatSpec with Matchers with BeforeAndAfterAll { +class ExecutorSystemLauncherSpec extends FlatSpec with Matchers with BeforeAndAfterAll { implicit var system: ActorSystem = null val workerId: WorkerId = WorkerId(0, 0L) val appId = 0 @@ -45,15 +46,15 @@ class ExecutorSystemLauncherSpec extends FlatSpec with Matchers with BeforeAndA val launchExecutorSystemTimeout = 3000 val activeConfig = TestUtil.DEFAULT_CONFIG. withValue(Constants.GEARPUMP_START_EXECUTOR_SYSTEM_TIMEOUT_MS, - ConfigValueFactory.fromAnyRef(launchExecutorSystemTimeout)) + ConfigValueFactory.fromAnyRef(launchExecutorSystemTimeout)) - override def beforeAll() = { + override def beforeAll(): Unit = { system = ActorSystem("test", activeConfig) } - override def afterAll() = { - system.shutdown() - system.awaitTermination() + override def afterAll(): Unit = { + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } it should "report success when worker launch the system successfully" in { @@ -90,14 +91,15 @@ class ExecutorSystemLauncherSpec extends FlatSpec with Matchers with BeforeAndA client.expectTerminated(launcher) } - it should "report timeout when trying to start a executor system on worker, and worker doesn't response" in { + it should "report timeout when trying to start a executor system on worker, " + + "and worker doesn't response" in { val client = TestProbe() val worker = TestProbe() val launcher = system.actorOf(Props(new ExecutorSystemLauncher(appId, session))) client.send(launcher, LaunchExecutorSystem(WorkerInfo(workerId, worker.ref), 0, Resource(1))) client.watch(launcher) val waitFor = launchExecutorSystemTimeout + 10000 - client.expectMsgType[LaunchExecutorSystemTimeout](waitFor milliseconds) - client.expectTerminated(launcher, waitFor milliseconds) + client.expectMsgType[LaunchExecutorSystemTimeout](waitFor.milliseconds) + client.expectTerminated(launcher, waitFor.milliseconds) } } diff --git a/core/src/test/scala/io/gearpump/cluster/appmaster/ExecutorSystemSchedulerSpec.scala b/core/src/test/scala/io/gearpump/cluster/appmaster/ExecutorSystemSchedulerSpec.scala index 106d38972..c2f6ee816 100644 --- a/core/src/test/scala/io/gearpump/cluster/appmaster/ExecutorSystemSchedulerSpec.scala +++ b/core/src/test/scala/io/gearpump/cluster/appmaster/ExecutorSystemSchedulerSpec.scala @@ -18,20 +18,22 @@ package io.gearpump.cluster.appmaster +import scala.concurrent.Await +import scala.concurrent.duration._ + import akka.actor.{Actor, ActorSystem, Props} import akka.testkit.TestProbe -import io.gearpump.WorkerId +import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} + import io.gearpump.cluster.AppMasterToMaster.RequestResource import io.gearpump.cluster.MasterToAppMaster.ResourceAllocated -import io.gearpump.cluster.{ExecutorJVMConfig, AppJar, TestUtil} import io.gearpump.cluster.appmaster.ExecutorSystemLauncher._ import io.gearpump.cluster.appmaster.ExecutorSystemScheduler._ import io.gearpump.cluster.appmaster.ExecutorSystemSchedulerSpec.{ExecutorSystemLauncherStarted, MockExecutorSystemLauncher} import io.gearpump.cluster.scheduler.{Resource, ResourceAllocation, ResourceRequest} +import io.gearpump.cluster.worker.WorkerId +import io.gearpump.cluster.{AppJar, TestUtil} import io.gearpump.jarstore.FilePath -import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} - -import scala.concurrent.duration._ class ExecutorSystemSchedulerSpec extends FlatSpec with Matchers with BeforeAndAfterEach { val appId = 0 @@ -43,7 +45,7 @@ class ExecutorSystemSchedulerSpec extends FlatSpec with Matchers with BeforeAndA val start = StartExecutorSystems(Array(resourceRequest), emptyJvmConfig) implicit var system: ActorSystem = null - var worker:TestProbe = null + var worker: TestProbe = null var workerInfo: WorkerInfo = null var masterProxy: TestProbe = null var launcher: TestProbe = null @@ -58,20 +60,21 @@ class ExecutorSystemSchedulerSpec extends FlatSpec with Matchers with BeforeAndA client = TestProbe() val scheduler = system.actorOf( - Props(new ExecutorSystemScheduler(appId, masterProxy.ref, - (appId: Int, session:Session) => Props(new MockExecutorSystemLauncher(launcher, session))))) + Props(new ExecutorSystemScheduler(appId, masterProxy.ref, (appId: Int, session: Session) => { + Props(new MockExecutorSystemLauncher(launcher, session)) + }))) client.send(scheduler, start) masterProxy.expectMsg(RequestResource(appId, resourceRequest)) } override def afterEach(): Unit = { - system.shutdown() - system.awaitTermination() + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } private def launcherStarted(launcher: TestProbe): Option[ExecutorSystemLauncherStarted] = { - val launcherStarted = launcher.receiveOne(15 seconds) + val launcherStarted = launcher.receiveOne(15.seconds) launcherStarted match { case start: ExecutorSystemLauncherStarted => Some(start) @@ -91,16 +94,18 @@ class ExecutorSystemSchedulerSpec extends FlatSpec with Matchers with BeforeAndA launcher.expectMsg(LaunchExecutorSystem(workerInfo, systemId, resource)) val executorSystemProbe = TestProbe() - val executorSystem = ExecutorSystem(systemId, null, executorSystemProbe.ref, resource, workerInfo) + val executorSystem = + ExecutorSystem(systemId, null, executorSystemProbe.ref, resource, workerInfo) launcher.reply(LaunchExecutorSystemSuccess(executorSystem, session)) client.expectMsg(ExecutorSystemStarted(executorSystem, Some(mockJar))) } it should "report failure when resource cannot be allocated" in { - client.expectMsg(30 seconds, StartExecutorSystemTimeout) + client.expectMsg(30.seconds, StartExecutorSystemTimeout) } - it should "schedule new resouce on new worker when target worker reject creating executor system" in { + it should "schedule new resouce on new worker " + + "when target worker reject creating executor system" in { masterProxy.reply(ResourceAllocated(Array(ResourceAllocation(resource, worker.ref, workerId)))) val ExecutorSystemLauncherStarted(session) = launcherStarted(launcher).get @@ -110,7 +115,8 @@ class ExecutorSystemSchedulerSpec extends FlatSpec with Matchers with BeforeAndA masterProxy.expectMsg(RequestResource(appId, resourceRequest)) } - it should "report failure when resource is allocated, but timeout when starting the executor system" in { + it should "report failure when resource is allocated, but timeout " + + "when starting the executor system" in { masterProxy.reply(ResourceAllocated(Array(ResourceAllocation(resource, worker.ref, workerId)))) val ExecutorSystemLauncherStarted(session) = launcherStarted(launcher).get diff --git a/core/src/test/scala/io/gearpump/cluster/appmaster/MasterConnectionKeeperSpec.scala b/core/src/test/scala/io/gearpump/cluster/appmaster/MasterConnectionKeeperSpec.scala index 9221ca353..3272b9922 100644 --- a/core/src/test/scala/io/gearpump/cluster/appmaster/MasterConnectionKeeperSpec.scala +++ b/core/src/test/scala/io/gearpump/cluster/appmaster/MasterConnectionKeeperSpec.scala @@ -18,51 +18,50 @@ package io.gearpump.cluster.appmaster +import scala.concurrent.Await +import scala.concurrent.duration._ + import akka.actor.{ActorRef, ActorSystem, Props} import akka.testkit.TestProbe -import io.gearpump.cluster.AppMasterToMaster.RegisterAppMaster -import io.gearpump.cluster.appmaster.MasterConnectionKeeper.MasterConnectionStatus.MasterConnected -import io.gearpump.cluster.master.MasterProxy.WatchMaster +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} + import io.gearpump.cluster.AppMasterToMaster.RegisterAppMaster import io.gearpump.cluster.MasterToAppMaster.AppMasterRegistered import io.gearpump.cluster.TestUtil -import io.gearpump.cluster.appmaster.MasterConnectionKeeper.MasterConnectionStatus._ +import io.gearpump.cluster.appmaster.MasterConnectionKeeper.MasterConnectionStatus.{MasterConnected, _} import io.gearpump.cluster.appmaster.MasterConnectionKeeperSpec.ConnectionKeeperTestEnv -import io.gearpump.cluster.master.MasterProxy import io.gearpump.cluster.master.MasterProxy.WatchMaster -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import scala.concurrent.duration._ - -class MasterConnectionKeeperSpec extends FlatSpec with Matchers with BeforeAndAfterAll { +class MasterConnectionKeeperSpec extends FlatSpec with Matchers with BeforeAndAfterAll { implicit var system: ActorSystem = null val register = RegisterAppMaster(null, null) val appId = 0 - override def beforeAll: Unit = { - system = ActorSystem("test", TestUtil.DEFAULT_CONFIG) - } + override def beforeAll(): Unit = { + system = ActorSystem("test", TestUtil.DEFAULT_CONFIG) + } - override def afterAll: Unit = { - system.shutdown() - system.awaitTermination() + override def afterAll(): Unit = { + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } private def startMasterConnectionKeeper: ConnectionKeeperTestEnv = { val statusChangeSubscriber = TestProbe() val master = TestProbe() - val keeper = system.actorOf(Props(new MasterConnectionKeeper(register, master.ref, statusChangeSubscriber.ref))) + val keeper = system.actorOf(Props( + new MasterConnectionKeeper(register, master.ref, statusChangeSubscriber.ref))) statusChangeSubscriber.watch(keeper) master.expectMsgType[WatchMaster] - //master is alive, response to RegisterAppMaster + // Master is alive, response to RegisterAppMaster master.expectMsgType[RegisterAppMaster] master.reply(AppMasterRegistered(appId)) - //notify listener that master is alive + // Notify listener that master is alive statusChangeSubscriber.expectMsg(MasterConnected) ConnectionKeeperTestEnv(master, keeper, statusChangeSubscriber) } @@ -75,23 +74,23 @@ class MasterConnectionKeeperSpec extends FlatSpec with Matchers with BeforeAndA import io.gearpump.cluster.master.MasterProxy.MasterRestarted val ConnectionKeeperTestEnv(master, keeper, masterChangeListener) = startMasterConnectionKeeper - //master is restarted + // Master is restarted master.send(keeper, MasterRestarted) master.expectMsgType[RegisterAppMaster] master.reply(AppMasterRegistered(appId)) masterChangeListener.expectMsg(MasterConnected) - //Recovery from Master restart is transparent to listener + // Recovery from Master restart is transparent to listener masterChangeListener.expectNoMsg() } it should "notify listener and then shutdown itself when master is dead" in { val ConnectionKeeperTestEnv(master, keeper, masterChangeListener) = startMasterConnectionKeeper - //master is dead + // Master is dead master.send(keeper, MasterStopped) - //keeper should tell the listener that master is stopped before shutting down itself + // Keeper should tell the listener that master is stopped before shutting down itself masterChangeListener.expectMsg(MasterStopped) masterChangeListener.expectTerminated(keeper) } @@ -100,18 +99,20 @@ class MasterConnectionKeeperSpec extends FlatSpec with Matchers with BeforeAndA val statusChangeSubscriber = TestProbe() val master = TestProbe() - //keeper will register to master by sending RegisterAppMaster - val keeper = system.actorOf(Props(new MasterConnectionKeeper(register, master.ref, statusChangeSubscriber.ref))) + // MasterConnectionKeeper register itself to master by sending RegisterAppMaster + val keeper = system.actorOf(Props(new MasterConnectionKeeper(register, + master.ref, statusChangeSubscriber.ref))) - //master doesn't reply to keeper, + // Master doesn't reply to keeper, statusChangeSubscriber.watch(keeper) - //timeout, keeper notify listener, and then make suicide - statusChangeSubscriber.expectMsg(60 seconds, MasterStopped) - statusChangeSubscriber.expectTerminated(keeper, 60 seconds) + // Timeout, keeper notify listener, and then make suicide + statusChangeSubscriber.expectMsg(60.seconds, MasterStopped) + statusChangeSubscriber.expectTerminated(keeper, 60.seconds) } } object MasterConnectionKeeperSpec { - case class ConnectionKeeperTestEnv(master: TestProbe, keeper: ActorRef, masterChangeListener: TestProbe) + case class ConnectionKeeperTestEnv( + master: TestProbe, keeper: ActorRef, masterChangeListener: TestProbe) } diff --git a/core/src/test/scala/io/gearpump/cluster/main/ArgumentParserSpec.scala b/core/src/test/scala/io/gearpump/cluster/main/ArgumentParserSpec.scala index f08e36812..7544f9a9e 100644 --- a/core/src/test/scala/io/gearpump/cluster/main/ArgumentParserSpec.scala +++ b/core/src/test/scala/io/gearpump/cluster/main/ArgumentParserSpec.scala @@ -30,7 +30,7 @@ class ArgumentParserSpec extends FlatSpec with Matchers { "opt2" -> CLIOption[Any]("", required = true)) } - val result = parser.parse(Array("-flag" , "-opt1", "1","-opt2", "2", "arg1", "arg2")) + val result = parser.parse(Array("-flag", "-opt1", "1", "-opt2", "2", "arg1", "arg2")) assert(result.getBoolean("flag")) assert(result.getInt("opt1") == 1) assert(result.getString("opt1") == "1") @@ -47,7 +47,7 @@ class ArgumentParserSpec extends FlatSpec with Matchers { "opt1" -> CLIOption[Any]("", required = true)) } - val result = parser.parse(Array("-opt1", "1","xx.MainClass", "-opt2", "2")) + val result = parser.parse(Array("-opt1", "1", "xx.MainClass", "-opt2", "2")) assert(result.getInt("opt1") == 1) assert(result.remainArgs.length == 3) diff --git a/core/src/test/scala/io/gearpump/cluster/master/AppMasterLauncherSpec.scala b/core/src/test/scala/io/gearpump/cluster/master/AppMasterLauncherSpec.scala index 8ff2ea1a8..2a8dba1a1 100644 --- a/core/src/test/scala/io/gearpump/cluster/master/AppMasterLauncherSpec.scala +++ b/core/src/test/scala/io/gearpump/cluster/master/AppMasterLauncherSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,24 +18,27 @@ package io.gearpump.cluster.master +import scala.util.Success + import akka.actor._ import akka.testkit.TestProbe -import io.gearpump.WorkerId +import com.typesafe.config.Config +import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} + import io.gearpump.cluster.AppMasterToMaster.RequestResource -import io.gearpump.cluster.AppMasterToWorker.{ShutdownExecutor, LaunchExecutor} +import io.gearpump.cluster.AppMasterToWorker.{LaunchExecutor, ShutdownExecutor} +import io.gearpump.cluster.MasterToAppMaster.ResourceAllocated import io.gearpump.cluster.MasterToClient.SubmitApplicationResult import io.gearpump.cluster.WorkerToAppMaster.ExecutorLaunchRejected -import io.gearpump.cluster.{TestUtil, MasterHarness} -import io.gearpump.cluster.MasterToAppMaster.ResourceAllocated -import io.gearpump.cluster.scheduler.{ResourceRequest, Resource, ResourceAllocation} +import io.gearpump.cluster.scheduler.{Resource, ResourceAllocation, ResourceRequest} +import io.gearpump.cluster.worker.WorkerId +import io.gearpump.cluster.{MasterHarness, TestUtil} import io.gearpump.util.ActorSystemBooter._ -import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} - -import scala.util.Success -class AppMasterLauncherSpec extends FlatSpec with Matchers with BeforeAndAfterEach with MasterHarness { +class AppMasterLauncherSpec extends FlatSpec with Matchers + with BeforeAndAfterEach with MasterHarness { - override def config = TestUtil.DEFAULT_CONFIG + override def config: Config = TestUtil.DEFAULT_CONFIG val appId = 1 val executorId = 2 @@ -45,7 +48,7 @@ class AppMasterLauncherSpec extends FlatSpec with Matchers with BeforeAndAfterEa var watcher: TestProbe = null var appMasterLauncher: ActorRef = null - override def beforeEach() = { + override def beforeEach(): Unit = { startActorSystem() master = createMockMaster() client = TestProbe()(getActorSystem) @@ -55,12 +58,13 @@ class AppMasterLauncherSpec extends FlatSpec with Matchers with BeforeAndAfterEa TestUtil.dummyApp, None, "username", master.ref, Some(client.ref))) watcher watch appMasterLauncher master.expectMsg(RequestResource(appId, ResourceRequest(Resource(1), WorkerId.unspecified))) - val resource = ResourceAllocated(Array(ResourceAllocation(Resource(1), worker.ref, WorkerId(0, 0L)))) + val resource = ResourceAllocated( + Array(ResourceAllocation(Resource(1), worker.ref, WorkerId(0, 0L)))) master.reply(resource) worker.expectMsgType[LaunchExecutor] } - override def afterEach() = { + override def afterEach(): Unit = { shutdownActorSystem() } @@ -79,7 +83,8 @@ class AppMasterLauncherSpec extends FlatSpec with Matchers with BeforeAndAfterEa worker.reply(ExecutorLaunchRejected("")) master.expectMsg(RequestResource(appId, ResourceRequest(Resource(1), WorkerId.unspecified))) - val resource = ResourceAllocated(Array(ResourceAllocation(Resource(1), worker.ref, WorkerId(0, 0L)))) + val resource = ResourceAllocated( + Array(ResourceAllocation(Resource(1), worker.ref, WorkerId(0, 0L)))) master.reply(resource) worker.expectMsgType[LaunchExecutor] diff --git a/core/src/test/scala/io/gearpump/cluster/master/ApplicationStateSpec.scala b/core/src/test/scala/io/gearpump/cluster/master/ApplicationStateSpec.scala index c1d6144bc..99cfc3788 100644 --- a/core/src/test/scala/io/gearpump/cluster/master/ApplicationStateSpec.scala +++ b/core/src/test/scala/io/gearpump/cluster/master/ApplicationStateSpec.scala @@ -18,11 +18,11 @@ package io.gearpump.cluster.master +import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} + import io.gearpump.cluster.appmaster.ApplicationState -import io.gearpump.cluster.{AppJar, AppDescription} -import org.scalatest.{BeforeAndAfterEach, Matchers, FlatSpec} -class ApplicationStateSpec extends FlatSpec with Matchers with BeforeAndAfterEach { +class ApplicationStateSpec extends FlatSpec with Matchers with BeforeAndAfterEach { "ApplicationState" should "check equal with respect to only appId and attemptId" in { val stateA = ApplicationState(0, "application0", 0, null, null, null, "A") diff --git a/core/src/test/scala/io/gearpump/cluster/master/MasterProxySpec.scala b/core/src/test/scala/io/gearpump/cluster/master/MasterProxySpec.scala index 690d8ad8a..b00712030 100644 --- a/core/src/test/scala/io/gearpump/cluster/master/MasterProxySpec.scala +++ b/core/src/test/scala/io/gearpump/cluster/master/MasterProxySpec.scala @@ -18,8 +18,7 @@ package io.gearpump.cluster.master -//TODO class MasterProxySpec { - // master proxy will retry to find the master + // Master proxy retries multiple times to find the master } diff --git a/core/src/test/scala/io/gearpump/cluster/master/MasterSpec.scala b/core/src/test/scala/io/gearpump/cluster/master/MasterSpec.scala index 972f9d54d..e4122da35 100644 --- a/core/src/test/scala/io/gearpump/cluster/master/MasterSpec.scala +++ b/core/src/test/scala/io/gearpump/cluster/master/MasterSpec.scala @@ -19,5 +19,4 @@ package io.gearpump.cluster.master class MasterSpec { - } diff --git a/core/src/test/scala/io/gearpump/metrics/MetricsSpec.scala b/core/src/test/scala/io/gearpump/metrics/MetricsSpec.scala index 1602d1043..3b3265ffe 100644 --- a/core/src/test/scala/io/gearpump/metrics/MetricsSpec.scala +++ b/core/src/test/scala/io/gearpump/metrics/MetricsSpec.scala @@ -18,13 +18,14 @@ package io.gearpump.metrics -import io.gearpump.codahale.metrics.{Counter => CodaHaleCounter, Histogram => CodaHaleHistogram, Meter => CodaHaleMeter} import org.mockito.Matchers._ import org.mockito.Mockito._ -import org.scalatest.{FlatSpec, Matchers} import org.scalatest.mock.MockitoSugar +import org.scalatest.{FlatSpec, Matchers} -class MetricsSpec extends FlatSpec with Matchers with MockitoSugar{ +import io.gearpump.codahale.metrics.{Counter => CodaHaleCounter, Histogram => CodaHaleHistogram, Meter => CodaHaleMeter} + +class MetricsSpec extends FlatSpec with Matchers with MockitoSugar { "Counter" should "handle sampleRate == 1" in { @@ -83,7 +84,6 @@ class MetricsSpec extends FlatSpec with Matchers with MockitoSugar{ verify(mockBaseHistogram, times(1)).update(6L) } - "Meter" should "handle sampleRate == 1" in { val mockBaseMeter = mock[CodaHaleMeter] @@ -101,7 +101,7 @@ class MetricsSpec extends FlatSpec with Matchers with MockitoSugar{ val mockBaseMeter = mock[CodaHaleMeter] - val meter = new Meter("m",mockBaseMeter, 2) + val meter = new Meter("m", mockBaseMeter, 2) meter.mark(1) meter.mark(3) diff --git a/core/src/test/scala/io/gearpump/partitioner/PartitionerSpec.scala b/core/src/test/scala/io/gearpump/partitioner/PartitionerSpec.scala index d5a3c180b..9509d94da 100644 --- a/core/src/test/scala/io/gearpump/partitioner/PartitionerSpec.scala +++ b/core/src/test/scala/io/gearpump/partitioner/PartitionerSpec.scala @@ -18,11 +18,12 @@ package io.gearpump.partitioner -import io.gearpump.Message import org.scalatest.{FlatSpec, Matchers} -class PartitionerSpec extends FlatSpec with Matchers { - val NUM = 10 +import io.gearpump.Message + +class PartitionerSpec extends FlatSpec with Matchers { + val NUM = 10 "HashPartitioner" should "hash same key to same slots" in { val partitioner = new HashPartitioner diff --git a/core/src/test/scala/io/gearpump/security/ConfigFileBasedAuthenticatorSpec.scala b/core/src/test/scala/io/gearpump/security/ConfigFileBasedAuthenticatorSpec.scala index c64cfa7d8..c38a55587 100644 --- a/core/src/test/scala/io/gearpump/security/ConfigFileBasedAuthenticatorSpec.scala +++ b/core/src/test/scala/io/gearpump/security/ConfigFileBasedAuthenticatorSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,20 +18,20 @@ package io.gearpump.security -import akka.actor.ActorSystem -import io.gearpump.cluster.TestUtil -import io.gearpump.security.Authenticator.AuthenticationResult -import org.scalatest.{Matchers, FlatSpec} - import scala.concurrent.Await import scala.concurrent.duration._ -class ConfigFileBasedAuthenticatorSpec extends FlatSpec with Matchers { +import akka.actor.ActorSystem +import org.scalatest.{FlatSpec, Matchers} + +import io.gearpump.cluster.TestUtil + +class ConfigFileBasedAuthenticatorSpec extends FlatSpec with Matchers { it should "authenticate correctly" in { val config = TestUtil.UI_CONFIG implicit val system = ActorSystem("ConfigFileBasedAuthenticatorSpec", config) implicit val ec = system.dispatcher - val timeout = 30 seconds + val timeout = 30.seconds val authenticator = new ConfigFileBasedAuthenticator(config) val guest = Await.result(authenticator.authenticate("guest", "guest", ec), timeout) @@ -48,7 +48,7 @@ class ConfigFileBasedAuthenticatorSpec extends FlatSpec with Matchers { assert(failedGuest == Authenticator.UnAuthenticated) assert(failedAdmin == Authenticator.UnAuthenticated) - system.shutdown() - system.awaitTermination() + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } } diff --git a/core/src/test/scala/io/gearpump/security/PasswordUtilSpec.scala b/core/src/test/scala/io/gearpump/security/PasswordUtilSpec.scala index 69abb80fb..4a3963bd5 100644 --- a/core/src/test/scala/io/gearpump/security/PasswordUtilSpec.scala +++ b/core/src/test/scala/io/gearpump/security/PasswordUtilSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,7 +18,7 @@ package io.gearpump.security -import org.scalatest.{Matchers, FlatSpec} +import org.scalatest.{FlatSpec, Matchers} class PasswordUtilSpec extends FlatSpec with Matchers { @@ -28,11 +28,10 @@ class PasswordUtilSpec extends FlatSpec with Matchers { val digest1 = PasswordUtil.hash(password) val digest2 = PasswordUtil.hash(password) - // we will use different salt each time, thus - // creating different hash. + // Uses different salt each time, thus creating different hash. assert(digest1 != digest2) - // both are valid hash. + // Both are valid hash. assert(PasswordUtil.verify(password, digest1)) assert(PasswordUtil.verify(password, digest2)) } diff --git a/core/src/test/scala/io/gearpump/serializer/SerializerSpec.scala b/core/src/test/scala/io/gearpump/serializer/SerializerSpec.scala index c18f4d521..3ed6ffa56 100644 --- a/core/src/test/scala/io/gearpump/serializer/SerializerSpec.scala +++ b/core/src/test/scala/io/gearpump/serializer/SerializerSpec.scala @@ -18,16 +18,19 @@ package io.gearpump.serializer +import scala.collection.JavaConverters._ +import scala.concurrent.Await +import scala.concurrent.duration.Duration + import akka.actor.{ActorSystem, ExtendedActorSystem} -import io.gearpump.esotericsoftware.kryo.io.{Input, Output} -import io.gearpump.esotericsoftware.kryo.{Kryo, Serializer => KryoSerializer} import com.typesafe.config.{ConfigFactory, ConfigValueFactory} -import io.gearpump.cluster.TestUtil -import io.gearpump.serializer.SerializerSpec._ import org.scalatest.mock.MockitoSugar import org.scalatest.{FlatSpec, Matchers} -import scala.collection.JavaConverters._ +import io.gearpump.cluster.TestUtil +import io.gearpump.esotericsoftware.kryo.io.{Input, Output} +import io.gearpump.esotericsoftware.kryo.{Kryo, Serializer => KryoSerializer} +import io.gearpump.serializer.SerializerSpec._ class SerializerSpec extends FlatSpec with Matchers with MockitoSugar { val config = ConfigFactory.empty.withValue("gearpump.serializers", @@ -56,8 +59,8 @@ class SerializerSpec extends FlatSpec with Matchers with MockitoSugar { val anotherA = serializer.deserialize(bytes) assert(anotherA.isInstanceOf[ClassA]) - system.shutdown() - system.awaitTermination() + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } } diff --git a/core/src/test/scala/io/gearpump/transport/MockTransportSerializer.scala b/core/src/test/scala/io/gearpump/transport/MockTransportSerializer.scala index 8582f29e8..71b42187d 100644 --- a/core/src/test/scala/io/gearpump/transport/MockTransportSerializer.scala +++ b/core/src/test/scala/io/gearpump/transport/MockTransportSerializer.scala @@ -15,15 +15,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package io.gearpump.transport import java.io.{DataInput, DataOutput} import io.gearpump.transport.MockTransportSerializer.NettyMessage import io.gearpump.transport.netty.ITransportMessageSerializer -import org.jboss.netty.buffer.ChannelBuffer -class MockTransportSerializer extends ITransportMessageSerializer{ +class MockTransportSerializer extends ITransportMessageSerializer { override def getLength(obj: scala.Any): Int = 4 override def serialize(dataOutput: DataOutput, transportMessage: scala.Any): Unit = { @@ -38,6 +38,6 @@ class MockTransportSerializer extends ITransportMessageSerializer{ } } -object MockTransportSerializer{ - case class NettyMessage(num : Int) +object MockTransportSerializer { + case class NettyMessage(num: Int) } diff --git a/core/src/test/scala/io/gearpump/transport/NettySpec.scala b/core/src/test/scala/io/gearpump/transport/NettySpec.scala index a81e21d40..6caf357e3 100644 --- a/core/src/test/scala/io/gearpump/transport/NettySpec.scala +++ b/core/src/test/scala/io/gearpump/transport/NettySpec.scala @@ -19,18 +19,19 @@ package io.gearpump.transport import java.util.concurrent.TimeUnit +import scala.concurrent.Await +import scala.concurrent.duration._ import akka.actor.{ActorRef, ActorSystem} import akka.testkit.TestProbe +import org.scalatest.mock.MockitoSugar +import org.scalatest.{FlatSpec, Matchers} + import io.gearpump.cluster.TestUtil import io.gearpump.transport.MockTransportSerializer.NettyMessage import io.gearpump.transport.netty.{Context, TaskMessage} -import org.scalatest.mock.MockitoSugar -import org.scalatest.{FlatSpec, Matchers} import io.gearpump.util.Util -import scala.concurrent.duration._ - class NettySpec extends FlatSpec with Matchers with MockitoSugar { "Netty Transport" should "send and receive message correctly " in { @@ -39,7 +40,7 @@ class NettySpec extends FlatSpec with Matchers with MockitoSugar { val context = new Context(system, conf) val serverActor = TestProbe()(system) - val port = Util.findFreePort + val port = Util.findFreePort() import system.dispatcher system.scheduler.scheduleOnce(Duration(1, TimeUnit.SECONDS)) { @@ -52,10 +53,10 @@ class NettySpec extends FlatSpec with Matchers with MockitoSugar { val data = NettyMessage(0) val msg = new TaskMessage(0, 1, 2, data) client ! msg - serverActor.expectMsg(15 seconds, data) - - context.close - system.shutdown() - system.awaitTermination() + serverActor.expectMsg(15.seconds, data) + + context.close() + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } } \ No newline at end of file diff --git a/core/src/test/scala/io/gearpump/util/ActorSystemBooterSpec.scala b/core/src/test/scala/io/gearpump/util/ActorSystemBooterSpec.scala index d5068fb14..13530ed6d 100644 --- a/core/src/test/scala/io/gearpump/util/ActorSystemBooterSpec.scala +++ b/core/src/test/scala/io/gearpump/util/ActorSystemBooterSpec.scala @@ -18,22 +18,24 @@ package io.gearpump.util +import scala.concurrent.Await +import scala.concurrent.duration.Duration + import akka.actor.{Actor, ActorSystem, Props} import akka.testkit.TestProbe -import io.gearpump.cluster.TestUtil -import io.gearpump.util.ActorSystemBooter.{ActorCreated, RegisterActorSystem} -import io.gearpump.cluster.TestUtil -import io.gearpump.util.ActorSystemBooter._ -import io.gearpump.util.ActorSystemBooterSpec._ import org.scalatest.mock.MockitoSugar import org.scalatest.{FlatSpec, Matchers} +import io.gearpump.cluster.TestUtil +import io.gearpump.util.ActorSystemBooter.{ActorCreated, RegisterActorSystem, _} +import io.gearpump.util.ActorSystemBooterSpec._ + class ActorSystemBooterSpec extends FlatSpec with Matchers with MockitoSugar { "ActorSystemBooter" should "report its address back" in { val boot = bootSystem() boot.prob.expectMsgType[RegisterActorSystem] - boot.shutdown + boot.shutdown() } "ActorSystemBooter" should "terminate itself when parent actor dies" in { @@ -44,9 +46,9 @@ class ActorSystemBooterSpec extends FlatSpec with Matchers with MockitoSugar { boot.prob.reply(ActorSystemRegistered(boot.prob.ref)) boot.prob.reply(BindLifeCycle(dummy)) boot.host.stop(dummy) - val terminated = retry(5)(boot.bootedSystem.isTerminated) + val terminated = retry(5)(boot.bootedSystem.whenTerminated.isCompleted) assert(terminated) - boot.shutdown + boot.shutdown() } "ActorSystemBooter" should "create new actor" in { @@ -59,10 +61,10 @@ class ActorSystemBooterSpec extends FlatSpec with Matchers with MockitoSugar { boot.prob.reply(CreateActor(Props(classOf[AcceptZeroArguments]), "zero")) boot.prob.expectMsgType[ActorCreated] - boot.shutdown + boot.shutdown() } - private def bootSystem() : Boot = { + private def bootSystem(): Boot = { val booter = ActorSystemBooter(TestUtil.DEFAULT_CONFIG) val system = ActorSystem("reportback", TestUtil.DEFAULT_CONFIG) @@ -75,16 +77,16 @@ class ActorSystemBooterSpec extends FlatSpec with Matchers with MockitoSugar { Boot(system, receiver, bootSystem) } - case class Boot(host : ActorSystem, prob : TestProbe, bootedSystem : ActorSystem) { - def shutdown = { - host.shutdown() - bootedSystem.shutdown() - host.awaitTermination() - bootedSystem.awaitTermination() + case class Boot(host: ActorSystem, prob: TestProbe, bootedSystem: ActorSystem) { + def shutdown(): Unit = { + host.terminate() + bootedSystem.terminate() + Await.result(host.whenTerminated, Duration.Inf) + Await.result(bootedSystem.whenTerminated, Duration.Inf) } } - def retry(seconds: Int)(fn: => Boolean) : Boolean = { + def retry(seconds: Int)(fn: => Boolean): Boolean = { val result = fn if (result) { result @@ -97,19 +99,19 @@ class ActorSystemBooterSpec extends FlatSpec with Matchers with MockitoSugar { object ActorSystemBooterSpec { class Dummy extends Actor { - def receive : Receive = { + def receive: Receive = { case _ => } } class AcceptZeroArguments extends Actor { - def receive : Receive = { + def receive: Receive = { case _ => } } - class AcceptThreeArguments(a : Int, b : Int, c : Int) extends Actor { - def receive : Receive = { + class AcceptThreeArguments(a: Int, b: Int, c: Int) extends Actor { + def receive: Receive = { case _ => } } diff --git a/core/src/test/scala/io/gearpump/util/ActorUtilSpec.scala b/core/src/test/scala/io/gearpump/util/ActorUtilSpec.scala index ec7281057..6ab5a2f36 100644 --- a/core/src/test/scala/io/gearpump/util/ActorUtilSpec.scala +++ b/core/src/test/scala/io/gearpump/util/ActorUtilSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,23 +18,23 @@ package io.gearpump.util +import org.scalatest.FlatSpec + import io.gearpump.transport.HostPort -import org.scalatest.mock.MockitoSugar -import org.scalatest.{Matchers, FlatSpec} -class ActorUtilSpec extends FlatSpec { - "masterActorPath" should "construct the ActorPath from HostPort" in { - import Constants.MASTER +class ActorUtilSpec extends FlatSpec { + "masterActorPath" should "construct the ActorPath from HostPort" in { + import io.gearpump.util.Constants.MASTER - val host = "127.0.0.1" - val port = 3000 - val master = HostPort("127.0.0.1", 3000) - val masterPath = ActorUtil.getMasterActorPath(master) - assert(masterPath.address.port == Some(port)) - assert(masterPath.address.system == MASTER) - assert(masterPath.address.host == Some(host)) - assert(masterPath.address.protocol == "akka.tcp") - assert(masterPath.toStringWithoutAddress == s"/user/$MASTER") - } + val host = "127.0.0.1" + val port = 3000 + val master = HostPort("127.0.0.1", 3000) + val masterPath = ActorUtil.getMasterActorPath(master) + assert(masterPath.address.port == Some(port)) + assert(masterPath.address.system == MASTER) + assert(masterPath.address.host == Some(host)) + assert(masterPath.address.protocol == "akka.tcp") + assert(masterPath.toStringWithoutAddress == s"/user/$MASTER") + } } diff --git a/core/src/test/scala/io/gearpump/util/ConfigsSpec.scala b/core/src/test/scala/io/gearpump/util/ConfigsSpec.scala index 07e812177..0c798f340 100644 --- a/core/src/test/scala/io/gearpump/util/ConfigsSpec.scala +++ b/core/src/test/scala/io/gearpump/util/ConfigsSpec.scala @@ -19,16 +19,20 @@ package io.gearpump.util import java.io.File +import scala.concurrent.Await +import scala.concurrent.duration.Duration import akka.actor.ActorSystem -import io.gearpump.cluster.{ClusterConfigSource, ClusterConfig, UserConfig} import org.scalatest.mock.MockitoSugar import org.scalatest.{FlatSpec, Matchers} -class ConfigsSpec extends FlatSpec with Matchers with MockitoSugar { +import io.gearpump.cluster.{ClusterConfig, ClusterConfigSource, UserConfig} + +class ConfigsSpec extends FlatSpec with Matchers with MockitoSugar { "Typesafe Cluster Configs" should "follow the override rules" in { - val conf = """ + val conf = + """ gearpump { gear = "gearpump" } @@ -40,7 +44,7 @@ class ConfigsSpec extends FlatSpec with Matchers with MockitoSugar { conf = "worker" } conf = "base" - """ + """ val file = File.createTempFile("test", ".conf") FileUtils.write(file, conf) @@ -68,8 +72,7 @@ class ConfigsSpec extends FlatSpec with Matchers with MockitoSugar { implicit val system = ActorSystem("forSerialization") - - val map = Map[String,String]("key1"->"1", "key2"->"value2") + val map = Map[String, String]("key1" -> "1", "key2" -> "value2") val user = new UserConfig(map) .withLong("key3", 2L) @@ -86,11 +89,11 @@ class ConfigsSpec extends FlatSpec with Matchers with MockitoSugar { val data = new ConfigsSpec.Data(3) assert(data == user.withValue("data", data).getValue[ConfigsSpec.Data]("data").get) - system.shutdown() - system.awaitTermination() + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } } -object ConfigsSpec{ +object ConfigsSpec { case class Data(value: Int) } \ No newline at end of file diff --git a/core/src/test/scala/io/gearpump/util/FileUtilsSpec.scala b/core/src/test/scala/io/gearpump/util/FileUtilsSpec.scala index 1c954d58e..30e42c72a 100644 --- a/core/src/test/scala/io/gearpump/util/FileUtilsSpec.scala +++ b/core/src/test/scala/io/gearpump/util/FileUtilsSpec.scala @@ -16,12 +16,13 @@ * limitations under the License. */ - package io.gearpump.util import java.io.File import java.util + import org.scalatest.FlatSpec + import io.gearpump.google.common.io.Files class FileUtilsSpec extends FlatSpec { @@ -43,7 +44,7 @@ class FileUtilsSpec extends FlatSpec { val file = File.createTempFile("fileutilspec", ".test") val bytes = TXT.toCharArray.map(_.toByte) FileUtils.writeByteArrayToFile(file, bytes) - util.Arrays.equals(bytes,FileUtils.readFileToByteArray(file)) + util.Arrays.equals(bytes, FileUtils.readFileToByteArray(file)) file.delete() } diff --git a/core/src/test/scala/io/gearpump/util/GraphSpec.scala b/core/src/test/scala/io/gearpump/util/GraphSpec.scala index 00284468b..20eab6d2d 100644 --- a/core/src/test/scala/io/gearpump/util/GraphSpec.scala +++ b/core/src/test/scala/io/gearpump/util/GraphSpec.scala @@ -20,9 +20,9 @@ package io.gearpump.util import org.scalacheck.Gen import org.scalatest.prop.PropertyChecks -import org.scalatest.{PropSpec, Matchers} +import org.scalatest.{Matchers, PropSpec} -import io.gearpump.util.Graph.{Path, Node} +import io.gearpump.util.Graph.{Node, Path} class GraphSpec extends PropSpec with PropertyChecks with Matchers { @@ -33,7 +33,7 @@ class GraphSpec extends PropSpec with PropertyChecks with Matchers { property("Graph with no edges should be built correctly") { val vertexSet = Set("A", "B", "C") - val graph = Graph(vertexSet.toSeq.map(Node):_ *) + val graph = Graph(vertexSet.toSeq.map(Node): _*) graph.vertices.toSet shouldBe vertexSet } @@ -107,7 +107,7 @@ class GraphSpec extends PropSpec with PropertyChecks with Matchers { val levelMap = graph.vertexHierarchyLevelMap() - //check whether the rule holds: : if vertex A -> B, then level(A) < level(B) + // Check whether the rule holds: : if vertex A -> B, then level(A) < level(B) levelMap("A") < levelMap("B") levelMap("A") < levelMap("C") levelMap("B") < levelMap("C") @@ -178,8 +178,8 @@ class GraphSpec extends PropSpec with PropertyChecks with Matchers { assert(graph.hasCycle()) } - property("topologicalOrderIterator " + - "and topologicalOrderWithCirclesIterator method should return equal order of graph with no circle") { + property("topologicalOrderIterator and topologicalOrderWithCirclesIterator method should " + + "return equal order of graph with no circle") { val graph = Graph(1 ~> 2 ~> 3, 4 ~> 2, 2 ~> 5) val topoNoCircles = graph.topologicalOrderIterator val topoWithCircles = graph.topologicalOrderWithCirclesIterator @@ -188,9 +188,10 @@ class GraphSpec extends PropSpec with PropertyChecks with Matchers { } property("Topological sort of graph with circles should work properly") { - val graph = Graph(0 ~> 1 ~> 3 ~> 4 ~> 6 ~> 5 ~> 7, 4 ~> 1, 1 ~> 2 ~> 4, 7 ~> 6, 8 ~> 2, 6 ~> 9, 4 ~> 10) + val graph = Graph(0 ~> 1 ~> 3 ~> 4 ~> 6 ~> 5 ~> 7, + 4 ~> 1, 1 ~> 2 ~> 4, 7 ~> 6, 8 ~> 2, 6 ~> 9, 4 ~> 10) val topoWithCircles = graph.topologicalOrderWithCirclesIterator - val trueTopoWithCircles = Iterator[Int](0, 8, 1, 3, 4, 2, 6 ,5, 7, 10, 9) + val trueTopoWithCircles = Iterator[Int](0, 8, 1, 3, 4, 2, 6, 5, 7, 10, 9) assert(trueTopoWithCircles.zip(topoWithCircles).forall(x => x._1 == x._2)) } diff --git a/core/src/test/scala/io/gearpump/util/TimeOutSchedulerSpec.scala b/core/src/test/scala/io/gearpump/util/TimeOutSchedulerSpec.scala index 213bd966a..ef362ffb6 100644 --- a/core/src/test/scala/io/gearpump/util/TimeOutSchedulerSpec.scala +++ b/core/src/test/scala/io/gearpump/util/TimeOutSchedulerSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,15 +15,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package io.gearpump.util +import scala.concurrent.duration._ + import akka.actor._ import akka.testkit.{ImplicitSender, TestActorRef, TestKit, TestProbe} -import io.gearpump.cluster.TestUtil -import io.gearpump.cluster.TestUtil import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} import org.slf4j.Logger -import scala.concurrent.duration._ + +import io.gearpump.cluster.TestUtil class TimeOutSchedulerSpec(_system: ActorSystem) extends TestKit(_system) with ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll { @@ -40,7 +42,7 @@ class TimeOutSchedulerSpec(_system: ActorSystem) extends TestKit(_system) with I val testActorRef = TestActorRef(Props(classOf[TestActor], mockActor.ref)) val testActor = testActorRef.underlyingActor.asInstanceOf[TestActor] testActor.sendMsgToIgnore() - mockActor.expectMsg(30 seconds, MessageTimeOut) + mockActor.expectMsg(30.seconds, MessageTimeOut) } } } @@ -60,7 +62,7 @@ class TestActor(mock: ActorRef) extends Actor with TimeOutScheduler { def sendMsgToIgnore(): Unit = { sendMsgWithTimeOutCallBack(target, Echo, 2000, sendMsgTimeOut()) } - + private def sendMsgTimeOut(): Unit = { mock ! MessageTimeOut } diff --git a/core/src/test/scala/io/gearpump/util/UtilSpec.scala b/core/src/test/scala/io/gearpump/util/UtilSpec.scala index 8d614fe42..b5bde04b1 100644 --- a/core/src/test/scala/io/gearpump/util/UtilSpec.scala +++ b/core/src/test/scala/io/gearpump/util/UtilSpec.scala @@ -18,17 +18,18 @@ package io.gearpump.util -import io.gearpump.transport.HostPort -import io.gearpump.util.Util._ import org.scalatest.mock.MockitoSugar import org.scalatest.{FlatSpec, Matchers} +import io.gearpump.transport.HostPort +import io.gearpump.util.Util._ + class UtilSpec extends FlatSpec with Matchers with MockitoSugar { it should "work" in { - assert(findFreePort.isSuccess) + assert(findFreePort().isSuccess) - assert(randInt != randInt) + assert(randInt() != randInt()) val hosts = parseHostList("host1:1,host2:2") assert(hosts(1) == HostPort("host2", 2)) diff --git a/daemon/src/main/resources/META-INF/services/io.gearpump.jarstore.JarStoreService b/daemon/src/main/resources/META-INF/services/io.gearpump.jarstore.JarStoreService index f0e0c5c90..d226af9cf 100644 --- a/daemon/src/main/resources/META-INF/services/io.gearpump.jarstore.JarStoreService +++ b/daemon/src/main/resources/META-INF/services/io.gearpump.jarstore.JarStoreService @@ -1,2 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + io.gearpump.jarstore.local.LocalJarStoreService io.gearpump.jarstore.dfs.DFSJarStoreService \ No newline at end of file diff --git a/daemon/src/main/scala/io/gearpump/cluster/DaemonMessage.scala b/daemon/src/main/scala/io/gearpump/cluster/DaemonMessage.scala index ac942ed12..420f1b67f 100644 --- a/daemon/src/main/scala/io/gearpump/cluster/DaemonMessage.scala +++ b/daemon/src/main/scala/io/gearpump/cluster/DaemonMessage.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,21 +18,34 @@ package io.gearpump.cluster import akka.actor.ActorRef -import io.gearpump.WorkerId + import io.gearpump.cluster.master.Master.MasterInfo import io.gearpump.cluster.scheduler.Resource +import io.gearpump.cluster.worker.WorkerId /** * Cluster Bootup Flow */ object WorkerToMaster { + + /** When an worker is started, it sends RegisterNewWorker */ case object RegisterNewWorker + + /** When worker lose connection with master, it tries to register itself again with old Id. */ case class RegisterWorker(workerId: WorkerId) + + /** Worker is responsible to broadcast its current status to master */ case class ResourceUpdate(worker: ActorRef, workerId: WorkerId, resource: Resource) } object MasterToWorker { - case class WorkerRegistered(workerId : WorkerId, masterInfo: MasterInfo) - case class UpdateResourceFailed(reason : String = null, ex: Throwable = null) + + /** Master confirm the reception of RegisterNewWorker or RegisterWorker */ + case class WorkerRegistered(workerId: WorkerId, masterInfo: MasterInfo) + + /** Worker have not received reply from master */ + case class UpdateResourceFailed(reason: String = null, ex: Throwable = null) + + /** Master is synced with worker on resource slots managed by current worker */ case object UpdateResourceSucceed } \ No newline at end of file diff --git a/daemon/src/main/scala/io/gearpump/cluster/embedded/EmbeddedCluster.scala b/daemon/src/main/scala/io/gearpump/cluster/embedded/EmbeddedCluster.scala index 6be52b205..53da645aa 100644 --- a/daemon/src/main/scala/io/gearpump/cluster/embedded/EmbeddedCluster.scala +++ b/daemon/src/main/scala/io/gearpump/cluster/embedded/EmbeddedCluster.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,16 +18,19 @@ package io.gearpump.cluster.embedded +import scala.collection.JavaConverters._ +import scala.concurrent.Await +import scala.concurrent.duration.Duration + import akka.actor.{ActorRef, ActorSystem, Props} -import com.typesafe.config.{ConfigValueFactory, Config} +import com.typesafe.config.{Config, ConfigValueFactory} + import io.gearpump.cluster.ClusterConfig import io.gearpump.cluster.client.ClientContext import io.gearpump.cluster.master.{Master => MasterActor} import io.gearpump.cluster.worker.{Worker => WorkerActor} -import io.gearpump.util.{LogUtil, Constants, Util, ActorUtil} -import io.gearpump.util.Constants.{GEARPUMP_METRIC_ENABLED, GEARPUMP_CLUSTER_EXECUTOR_WORKER_SHARE_SAME_PROCESS, MASTER, GEARPUMP_CLUSTER_MASTERS} -import scala.collection.JavaConverters._ - +import io.gearpump.util.Constants.{GEARPUMP_CLUSTER_EXECUTOR_WORKER_SHARE_SAME_PROCESS, GEARPUMP_CLUSTER_MASTERS, GEARPUMP_METRIC_ENABLED, MASTER} +import io.gearpump.util.{LogUtil, Util} /** * Create a in-process cluster with single worker @@ -41,8 +44,8 @@ class EmbeddedCluster(inputConfig: Config) { private val LOG = LogUtil.getLogger(getClass) - def start: Unit = { - val port = Util.findFreePort.get + def start(): Unit = { + val port = Util.findFreePort().get val akkaConf = getConfig(inputConfig, port) _config = akkaConf val system = ActorSystem(MASTER, akkaConf) @@ -64,10 +67,13 @@ class EmbeddedCluster(inputConfig: Config) { private def getConfig(inputConfig: Config, port: Int): Config = { val config = inputConfig. withValue("akka.remote.netty.tcp.port", ConfigValueFactory.fromAnyRef(port)). - withValue(GEARPUMP_CLUSTER_MASTERS, ConfigValueFactory.fromIterable(List(s"127.0.0.1:$port").asJava)). - withValue(GEARPUMP_CLUSTER_EXECUTOR_WORKER_SHARE_SAME_PROCESS, ConfigValueFactory.fromAnyRef(true)). + withValue(GEARPUMP_CLUSTER_MASTERS, + ConfigValueFactory.fromIterable(List(s"127.0.0.1:$port").asJava)). + withValue(GEARPUMP_CLUSTER_EXECUTOR_WORKER_SHARE_SAME_PROCESS, + ConfigValueFactory.fromAnyRef(true)). withValue(GEARPUMP_METRIC_ENABLED, ConfigValueFactory.fromAnyRef(true)). - withValue("akka.actor.provider", ConfigValueFactory.fromAnyRef("akka.cluster.ClusterActorRefProvider")) + withValue("akka.actor.provider", + ConfigValueFactory.fromAnyRef("akka.cluster.ClusterActorRefProvider")) config } @@ -75,14 +81,14 @@ class EmbeddedCluster(inputConfig: Config) { ClientContext(_config, _system, _master) } - def stop: Unit = { + def stop(): Unit = { _system.stop(_master) - _system.shutdown() - _system.awaitTermination() + _system.terminate() + Await.result(_system.whenTerminated, Duration.Inf) } } -object EmbeddedCluster{ +object EmbeddedCluster { def apply(): EmbeddedCluster = { new EmbeddedCluster(ClusterConfig.master()) } diff --git a/daemon/src/main/scala/io/gearpump/cluster/main/AppSubmitter.scala b/daemon/src/main/scala/io/gearpump/cluster/main/AppSubmitter.scala index 90f653cfe..68f778e49 100644 --- a/daemon/src/main/scala/io/gearpump/cluster/main/AppSubmitter.scala +++ b/daemon/src/main/scala/io/gearpump/cluster/main/AppSubmitter.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,13 +20,13 @@ package io.gearpump.cluster.main import java.io.File import java.net.{URL, URLClassLoader} import java.util.jar.JarFile +import scala.util.Try -import io.gearpump.util.{Util, Constants} -import io.gearpump.util.{AkkaApp, Constants, LogUtil, Util} import org.slf4j.Logger -import scala.util.Try +import io.gearpump.util.{AkkaApp, Constants, LogUtil, Util} +/** Tool to submit an application jar to cluster */ object AppSubmitter extends AkkaApp with ArgumentsParser { val LOG: Logger = LogUtil.getLogger(getClass) @@ -35,60 +35,66 @@ object AppSubmitter extends AkkaApp with ArgumentsParser { override val description = "Submit an application to Master by providing a jar" override val options: Array[(String, CLIOption[Any])] = Array( - "namePrefix" -> CLIOption[String]("", required = false, defaultValue = Some("")), + "namePrefix" -> CLIOption[String]("", required = false, + defaultValue = Some("")), "jar" -> CLIOption(".jar", required = true), - "executors" -> CLIOption[Int]("number of executor to launch", required = false, defaultValue = Some(1)), - "verbose" -> CLIOption("", required = false, defaultValue = Some(false)), + "executors" -> CLIOption[Int]("number of executor to launch", required = false, + defaultValue = Some(1)), + "verbose" -> CLIOption("", required = false, + defaultValue = Some(false)), // For document purpose only, OPTION_CONFIG option is not used here. // OPTION_CONFIG is parsed by parent shell command "Gear" transparently. - Gear.OPTION_CONFIG -> CLIOption("custom configuration file", required = false, defaultValue = None)) + Gear.OPTION_CONFIG -> CLIOption("custom configuration file", required = false, + defaultValue = None)) def main(akkaConf: Config, args: Array[String]): Unit = { val config = parse(args) - if (null == config) { - return - } + if (null != config) { - val verbose = config.getBoolean("verbose") - if (verbose) { - LogUtil.verboseLogToConsole - } + val verbose = config.getBoolean("verbose") + if (verbose) { + LogUtil.verboseLogToConsole() + } - val jar = config.getString("jar") + val jar = config.getString("jar") - // Set jar path to be submitted to cluster - System.setProperty(Constants.GEARPUMP_APP_JAR, jar) - System.setProperty(Constants.APPLICATION_EXECUTOR_NUMBER, config.getInt("executors").toString) + // Set jar path to be submitted to cluster + System.setProperty(Constants.GEARPUMP_APP_JAR, jar) + System.setProperty(Constants.APPLICATION_EXECUTOR_NUMBER, config.getInt("executors").toString) - val namePrefix = config.getString("namePrefix") - if (namePrefix.nonEmpty) { - if (!Util.validApplicationName(namePrefix)) { - throw new Exception(s"$namePrefix is not a valid prefix for an application name") + val namePrefix = config.getString("namePrefix") + if (namePrefix.nonEmpty) { + if (!Util.validApplicationName(namePrefix)) { + throw new Exception(s"$namePrefix is not a valid prefix for an application name") + } + System.setProperty(Constants.GEARPUMP_APP_NAME_PREFIX, namePrefix) } - System.setProperty(Constants.GEARPUMP_APP_NAME_PREFIX, namePrefix) - } - val jarFile = new java.io.File(jar) + val jarFile = new java.io.File(jar) - //start main class - if (!jarFile.exists()) { - throw new Exception(s"jar $jar does not exist") - } + // Start main class + if (!jarFile.exists()) { + throw new Exception(s"jar $jar does not exist") + } - val classLoader: URLClassLoader = new URLClassLoader(Array(new URL("file:" + jarFile.getAbsolutePath)), - Thread.currentThread().getContextClassLoader()) - val (main, arguments) = parseMain(jarFile, config.remainArgs, classLoader) + val classLoader: URLClassLoader = new URLClassLoader(Array(new URL("file:" + + jarFile.getAbsolutePath)), Thread.currentThread().getContextClassLoader()) + val (main, arguments) = parseMain(jarFile, config.remainArgs, classLoader) - //set the context classloader as ActorSystem will use context classloader in precedence. - Thread.currentThread().setContextClassLoader(classLoader) - val clazz = classLoader.loadClass(main) - val mainMethod = clazz.getMethod("main", classOf[Array[String]]) - mainMethod.invoke(null, arguments) + // Set to context classloader. ActorSystem pick context classloader in preference + Thread.currentThread().setContextClassLoader(classLoader) + val clazz = classLoader.loadClass(main) + val mainMethod = clazz.getMethod("main", classOf[Array[String]]) + mainMethod.invoke(null, arguments) + } } - private def parseMain(jar: File, remainArgs: Array[String], classLoader: ClassLoader): (String, Array[String]) = { - val mainInManifest = Option(new JarFile(jar).getManifest.getMainAttributes.getValue("Main-Class")).getOrElse("") + private def parseMain(jar: File, remainArgs: Array[String], classLoader: ClassLoader) + : (String, Array[String]) = { + val mainInManifest = Option(new JarFile(jar).getManifest.getMainAttributes. + getValue("Main-Class")).getOrElse("") + if (remainArgs.length > 0 && Try(classLoader.loadClass(remainArgs(0))).isSuccess) { (remainArgs(0), remainArgs.drop(1)) } else if (mainInManifest.nonEmpty) { diff --git a/daemon/src/main/scala/io/gearpump/cluster/main/Gear.scala b/daemon/src/main/scala/io/gearpump/cluster/main/Gear.scala index 31b749202..442372720 100644 --- a/daemon/src/main/scala/io/gearpump/cluster/main/Gear.scala +++ b/daemon/src/main/scala/io/gearpump/cluster/main/Gear.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,10 +17,11 @@ */ package io.gearpump.cluster.main -import io.gearpump.util.{Constants, LogUtil} import org.slf4j.Logger -object Gear { +import io.gearpump.util.{Constants, LogUtil} + +object Gear { val OPTION_CONFIG = "conf" @@ -29,12 +30,14 @@ object Gear { val commands = Map("app" -> AppSubmitter, "kill" -> Kill, "info" -> Info, "replay" -> Replay, "main" -> MainRunner) - def usage: Unit = { + def usage(): Unit = { val keys = commands.keys.toList.sorted + // scalastyle:off println Console.err.println("Usage: " + "<" + keys.mkString("|") + ">") + // scalastyle:on println } - def executeCommand(command : String, commandArgs : Array[String]) = { + private def executeCommand(command: String, commandArgs: Array[String]) = { commands.get(command).map(_.main(commandArgs)) if (!commands.contains(command)) { val allArgs = (command +: commandArgs.toList).toArray @@ -42,15 +45,15 @@ object Gear { } } - def main(inputArgs: Array[String]) = { + def main(inputArgs: Array[String]): Unit = { val (configFile, args) = extractConfig(inputArgs) if (configFile != null) { - // set custom config file... + // Sets custom config file... System.setProperty(Constants.GEARPUMP_CUSTOM_CONFIG_FILE, configFile) } if (args.length == 0) { - usage + usage() } else { val command = args(0) val commandArgs = args.drop(1) @@ -62,7 +65,7 @@ object Gear { var index = 0 var result = List.empty[String] - var configFile:String = null + var configFile: String = null while (index < inputArgs.length) { val item = inputArgs(index) if (item == s"-$OPTION_CONFIG") { diff --git a/daemon/src/main/scala/io/gearpump/cluster/main/Info.scala b/daemon/src/main/scala/io/gearpump/cluster/main/Info.scala index 878bcbfc4..4922690bd 100644 --- a/daemon/src/main/scala/io/gearpump/cluster/main/Info.scala +++ b/daemon/src/main/scala/io/gearpump/cluster/main/Info.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,13 +17,13 @@ */ package io.gearpump.cluster.main +import org.slf4j.Logger + import io.gearpump.cluster.MasterToAppMaster.AppMastersData import io.gearpump.cluster.client.ClientContext import io.gearpump.util.{AkkaApp, LogUtil} -import org.slf4j.Logger - -import scala.util.Try +/** Tool to query master info */ object Info extends AkkaApp with ArgumentsParser { private val LOG: Logger = LogUtil.getLogger(getClass) @@ -31,11 +31,12 @@ object Info extends AkkaApp with ArgumentsParser { override val options: Array[(String, CLIOption[Any])] = Array( // For document purpose only, OPTION_CONFIG option is not used here. // OPTION_CONFIG is parsed by parent shell command "Gear" transparently. - Gear.OPTION_CONFIG -> CLIOption("custom configuration file", required = false, defaultValue = None)) - + Gear.OPTION_CONFIG -> CLIOption("custom configuration file", required = false, + defaultValue = None)) override val description = "Query the Application list" + // scalastyle:off println def main(akkaConf: Config, args: Array[String]): Unit = { val client = ClientContext(akkaConf) @@ -48,4 +49,5 @@ object Info extends AkkaApp with ArgumentsParser { } client.close() } + // scalastyle:on println } \ No newline at end of file diff --git a/daemon/src/main/scala/io/gearpump/cluster/main/Kill.scala b/daemon/src/main/scala/io/gearpump/cluster/main/Kill.scala index 195809c9f..3ce781f84 100644 --- a/daemon/src/main/scala/io/gearpump/cluster/main/Kill.scala +++ b/daemon/src/main/scala/io/gearpump/cluster/main/Kill.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,12 +18,12 @@ package io.gearpump.cluster.main -import io.gearpump.cluster.client.ClientContext -import io.gearpump.util.{AkkaApp, LogUtil} import org.slf4j.Logger -import scala.util.Try +import io.gearpump.cluster.client.ClientContext +import io.gearpump.util.{AkkaApp, LogUtil} +/** Tool to kill an App */ object Kill extends AkkaApp with ArgumentsParser { private val LOG: Logger = LogUtil.getLogger(getClass) @@ -32,20 +32,19 @@ object Kill extends AkkaApp with ArgumentsParser { "appid" -> CLIOption("", required = true), // For document purpose only, OPTION_CONFIG option is not used here. // OPTION_CONFIG is parsed by parent shell command "Gear" transparently. - Gear.OPTION_CONFIG -> CLIOption("custom configuration file", required = false, defaultValue = None)) + Gear.OPTION_CONFIG -> CLIOption("custom configuration file", required = false, + defaultValue = None)) override val description = "Kill an application with application Id" def main(akkaConf: Config, args: Array[String]): Unit = { val config = parse(args) - if (null == config) { - return + if (null != config) { + val client = ClientContext(akkaConf) + LOG.info("Client ") + client.shutdown(config.getInt("appid")) + client.close() } - - val client = ClientContext(akkaConf) - LOG.info("Client ") - client.shutdown(config.getInt("appid")) - client.close() } } diff --git a/daemon/src/main/scala/io/gearpump/cluster/main/Local.scala b/daemon/src/main/scala/io/gearpump/cluster/main/Local.scala index d6b2479d0..d5681dfc9 100644 --- a/daemon/src/main/scala/io/gearpump/cluster/main/Local.scala +++ b/daemon/src/main/scala/io/gearpump/cluster/main/Local.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,17 +18,20 @@ package io.gearpump.cluster.main +import scala.collection.JavaConverters._ +import scala.concurrent.Await +import scala.concurrent.duration.Duration + import akka.actor.{ActorSystem, Props} import com.typesafe.config.ConfigValueFactory +import org.slf4j.Logger + import io.gearpump.cluster.ClusterConfig import io.gearpump.cluster.master.{Master => MasterActor} import io.gearpump.cluster.worker.{Worker => WorkerActor} import io.gearpump.util.Constants._ import io.gearpump.util.LogUtil.ProcessType -import io.gearpump.util.{AkkaApp, ActorUtil, Constants, LogUtil, Util} -import org.slf4j.Logger - -import scala.collection.JavaConversions._ +import io.gearpump.util.{ActorUtil, AkkaApp, Constants, LogUtil, Util} object Local extends AkkaApp with ArgumentsParser { override def akkaConfig: Config = ClusterConfig.master() @@ -37,7 +40,8 @@ object Local extends AkkaApp with ArgumentsParser { override val options: Array[(String, CLIOption[Any])] = Array("sameprocess" -> CLIOption[Boolean]("", required = false, defaultValue = Some(false)), - "workernum"-> CLIOption[Int]("", required = false, defaultValue = Some(2))) + "workernum" -> CLIOption[Int]("", required = false, + defaultValue = Some(2))) override val description = "Start a local cluster" @@ -49,23 +53,23 @@ object Local extends AkkaApp with ArgumentsParser { } val config = parse(args) - if (null == config) { - return + if (null != config) { + local(config.getInt("workernum"), config.getBoolean("sameprocess"), akkaConf) } - local(config.getInt("workernum"), config.getBoolean("sameprocess"), akkaConf) } - def local(workerCount : Int, sameProcess : Boolean, akkaConf: Config) : Unit = { + def local(workerCount: Int, sameProcess: Boolean, akkaConf: Config): Unit = { if (sameProcess) { LOG.info("Starting local in same process") System.setProperty("LOCAL", "true") } - val masters = akkaConf.getStringList(Constants.GEARPUMP_CLUSTER_MASTERS).toList.flatMap(Util.parseHostList) + val masters = akkaConf.getStringList(Constants.GEARPUMP_CLUSTER_MASTERS) + .asScala.flatMap(Util.parseHostList) val local = akkaConf.getString(Constants.GEARPUMP_HOSTNAME) - if(masters.size != 1 && masters.head.host != local) { - LOG.error(s"The ${Constants.GEARPUMP_CLUSTER_MASTERS} is not match with ${Constants.GEARPUMP_HOSTNAME}") - + if (masters.size != 1 && masters.head.host != local) { + LOG.error(s"The ${Constants.GEARPUMP_CLUSTER_MASTERS} is not match " + + s"with ${Constants.GEARPUMP_HOSTNAME}") } else { val hostPort = masters.head @@ -80,7 +84,7 @@ object Local extends AkkaApp with ArgumentsParser { system.actorOf(Props(classOf[WorkerActor], master), classOf[WorkerActor].getSimpleName + id) } - system.awaitTermination() + Await.result(system.whenTerminated, Duration.Inf) } } } diff --git a/daemon/src/main/scala/io/gearpump/cluster/main/MainRunner.scala b/daemon/src/main/scala/io/gearpump/cluster/main/MainRunner.scala index ea235ed64..923a6467d 100644 --- a/daemon/src/main/scala/io/gearpump/cluster/main/MainRunner.scala +++ b/daemon/src/main/scala/io/gearpump/cluster/main/MainRunner.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,19 +18,19 @@ package io.gearpump.cluster.main -import io.gearpump.util.{LogUtil, AkkaApp} -import io.gearpump.util.{AkkaApp, LogUtil} import org.slf4j.Logger -import scala.util.Try +import io.gearpump.util.{AkkaApp, LogUtil} +/** Tool to run any main class by providing a jar */ object MainRunner extends AkkaApp with ArgumentsParser { private val LOG: Logger = LogUtil.getLogger(getClass) override val options: Array[(String, CLIOption[Any])] = Array( // For document purpose only, OPTION_CONFIG option is not used here. // OPTION_CONFIG is parsed by parent shell command "Gear" transparently. - Gear.OPTION_CONFIG -> CLIOption("custom configuration file", required = false, defaultValue = None)) + Gear.OPTION_CONFIG -> CLIOption("custom configuration file", required = false, + defaultValue = None)) def main(akkaConf: Config, args: Array[String]): Unit = { val mainClazz = args(0) diff --git a/daemon/src/main/scala/io/gearpump/cluster/main/Master.scala b/daemon/src/main/scala/io/gearpump/cluster/main/Master.scala index 8d4515a61..6a4ac0762 100644 --- a/daemon/src/main/scala/io/gearpump/cluster/main/Master.scala +++ b/daemon/src/main/scala/io/gearpump/cluster/main/Master.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,23 +19,24 @@ package io.gearpump.cluster.main import java.util.concurrent.TimeUnit +import scala.collection.JavaConverters._ +import scala.collection.immutable +import scala.concurrent.Await +import scala.concurrent.duration._ import akka.actor._ import akka.cluster.ClusterEvent._ import akka.cluster.ddata.DistributedData +import akka.cluster.singleton.{ClusterSingletonManager, ClusterSingletonManagerSettings, ClusterSingletonProxy, ClusterSingletonProxySettings} import akka.cluster.{Cluster, Member, MemberStatus} -import akka.cluster.singleton.{ClusterSingletonManagerSettings, ClusterSingletonProxySettings, ClusterSingletonManager, ClusterSingletonProxy} import com.typesafe.config.ConfigValueFactory +import org.slf4j.Logger + import io.gearpump.cluster.ClusterConfig import io.gearpump.cluster.master.{Master => MasterActor} import io.gearpump.util.Constants._ import io.gearpump.util.LogUtil.ProcessType import io.gearpump.util.{AkkaApp, Constants, LogUtil} -import org.slf4j.Logger - -import scala.collection.JavaConverters._ -import scala.collection.immutable -import scala.concurrent.duration._ object Master extends AkkaApp with ArgumentsParser { @@ -44,14 +45,14 @@ object Master extends AkkaApp with ArgumentsParser { override def akkaConfig: Config = ClusterConfig.master() override val options: Array[(String, CLIOption[Any])] = - Array("ip"->CLIOption[String]("",required = true), - "port"->CLIOption("",required = true)) + Array("ip" -> CLIOption[String]("", required = true), + "port" -> CLIOption("", required = true)) override val description = "Start Master daemon" def main(akkaConf: Config, args: Array[String]): Unit = { - this.LOG = { + this.LOG = { LogUtil.loadConfiguration(akkaConf, ProcessType.MASTER) LogUtil.getLogger(getClass) } @@ -60,27 +61,29 @@ object Master extends AkkaApp with ArgumentsParser { master(config.getString("ip"), config.getInt("port"), akkaConf) } - def verifyMaster(master : String, port: Int, masters : Iterable[String]) = { - masters.exists{ hostPort => + private def verifyMaster(master: String, port: Int, masters: Iterable[String]) = { + masters.exists { hostPort => hostPort == s"$master:$port" } } - def master(ip:String, port : Int, akkaConf: Config): Unit = { + private def master(ip: String, port: Int, akkaConf: Config): Unit = { val masters = akkaConf.getStringList(Constants.GEARPUMP_CLUSTER_MASTERS).asScala if (!verifyMaster(ip, port, masters)) { - LOG.error(s"The provided ip $ip and port $port doesn't conform with config at gearpump.cluster.masters: ${masters.mkString(", ")}") + LOG.error(s"The provided ip $ip and port $port doesn't conform with config at " + + s"gearpump.cluster.masters: ${masters.mkString(", ")}") System.exit(-1) } val masterList = masters.map(master => s"akka.tcp://${MASTER}@$master").toList.asJava - val quorum = masterList.size() /2 + 1 + val quorum = masterList.size() / 2 + 1 val masterConfig = akkaConf. withValue("akka.remote.netty.tcp.port", ConfigValueFactory.fromAnyRef(port)). withValue(NETTY_TCP_HOSTNAME, ConfigValueFactory.fromAnyRef(ip)). withValue("akka.cluster.seed-nodes", ConfigValueFactory.fromAnyRef(masterList)). - withValue(s"akka.cluster.role.${MASTER}.min-nr-of-members", ConfigValueFactory.fromAnyRef(quorum)) + withValue(s"akka.cluster.role.${MASTER}.min-nr-of-members", + ConfigValueFactory.fromAnyRef(quorum)) LOG.info(s"Starting Master Actor system $ip:$port, master list: ${masters.mkString(";")}") val system = ActorSystem(MASTER, masterConfig) @@ -88,19 +91,21 @@ object Master extends AkkaApp with ArgumentsParser { val replicator = DistributedData(system).replicator LOG.info(s"Replicator path: ${replicator.path}") - //start singleton manager + // Starts singleton manager val singletonManager = system.actorOf(ClusterSingletonManager.props( singletonProps = Props(classOf[MasterWatcher], MASTER), terminationMessage = PoisonPill, - settings = ClusterSingletonManagerSettings(system).withSingletonName(MASTER_WATCHER).withRole(MASTER)), + settings = ClusterSingletonManagerSettings(system).withSingletonName(MASTER_WATCHER) + .withRole(MASTER)), name = SINGLETON_MANAGER) - //start master proxy + // Start master proxy val masterProxy = system.actorOf(ClusterSingletonProxy.props( singletonManagerPath = s"/user/${SINGLETON_MANAGER}", // The effective singleton is s"${MASTER_WATCHER}/$MASTER" instead of s"${MASTER_WATCHER}". - // Master will only be created when there is a majority of machines started. - settings = ClusterSingletonProxySettings(system).withSingletonName(s"${MASTER_WATCHER}/$MASTER").withRole(MASTER)), + // Master is created when there is a majority of machines started. + settings = ClusterSingletonProxySettings(system) + .withSingletonName(s"${MASTER_WATCHER}/$MASTER").withRole(MASTER)), name = MASTER ) @@ -108,8 +113,8 @@ object Master extends AkkaApp with ArgumentsParser { val mainThread = Thread.currentThread() Runtime.getRuntime().addShutdownHook(new Thread() { - override def run() : Unit = { - if (!system.isTerminated) { + override def run(): Unit = { + if (!system.whenTerminated.isCompleted) { LOG.info("Triggering shutdown hook....") system.stop(masterProxy) @@ -117,21 +122,21 @@ object Master extends AkkaApp with ArgumentsParser { cluster.leave(cluster.selfAddress) cluster.down(cluster.selfAddress) try { - system.awaitTermination(Duration(3, TimeUnit.SECONDS)) + Await.result(system.whenTerminated, Duration(3, TimeUnit.SECONDS)) } catch { - case ex : Exception => //ignore + case ex: Exception => // Ignore } - system.shutdown() + system.terminate() mainThread.join() } } }) - system.awaitTermination() + Await.result(system.whenTerminated, Duration.Inf) } } -class MasterWatcher(role: String) extends Actor with ActorLogging { +class MasterWatcher(role: String) extends Actor with ActorLogging { import context.dispatcher val cluster = Cluster(context.system) @@ -142,13 +147,13 @@ class MasterWatcher(role: String) extends Actor with ActorLogging { val system = context.system - // sort by age, oldest first + // Sorts by age, oldest first val ageOrdering = Ordering.fromLessThan[Member] { (a, b) => a.isOlderThan(b) } var membersByAge: immutable.SortedSet[Member] = immutable.SortedSet.empty(ageOrdering) - def receive : Receive = null + def receive: Receive = null - // subscribe to MemberEvent, re-subscribe when restart + // Subscribes to MemberEvent, re-subscribe when restart override def preStart(): Unit = { cluster.subscribe(self, classOf[MemberEvent]) context.become(waitForInit) @@ -159,14 +164,15 @@ class MasterWatcher(role: String) extends Actor with ActorLogging { def matchingRole(member: Member): Boolean = member.hasRole(role) - def waitForInit : Receive = { + def waitForInit: Receive = { case state: CurrentClusterState => { membersByAge = immutable.SortedSet.empty(ageOrdering) ++ state.members.filter(m => m.status == MemberStatus.Up && matchingRole(m)) if (membersByAge.size < quorum) { membersByAge.iterator.mkString(",") - log.info(s"We cannot get a quorum, $quorum, shutting down...${membersByAge.iterator.mkString(",")}") + log.info(s"We cannot get a quorum, $quorum, " + + s"shutting down...${membersByAge.iterator.mkString(",")}") context.become(waitForShutdown) self ! MasterWatcher.Shutdown } else { @@ -176,34 +182,36 @@ class MasterWatcher(role: String) extends Actor with ActorLogging { } } - def waitForClusterEvent : Receive = { - case MemberUp(m) if matchingRole(m) => { + def waitForClusterEvent: Receive = { + case MemberUp(m) if matchingRole(m) => { membersByAge += m } - case mEvent: MemberEvent if (mEvent.isInstanceOf[MemberExited] || mEvent.isInstanceOf[MemberRemoved]) && matchingRole(mEvent.member) => { + case mEvent: MemberEvent if (mEvent.isInstanceOf[MemberExited] || + mEvent.isInstanceOf[MemberRemoved]) && matchingRole(mEvent.member) => { log.info(s"member removed ${mEvent.member}") val m = mEvent.member membersByAge -= m if (membersByAge.size < quorum) { - log.info(s"We cannot get a quorum, $quorum, shutting down...${membersByAge.iterator.mkString(",")}") + log.info(s"We cannot get a quorum, $quorum, " + + s"shutting down...${membersByAge.iterator.mkString(",")}") context.become(waitForShutdown) self ! MasterWatcher.Shutdown } } } - def waitForShutdown : Receive = { + def waitForShutdown: Receive = { case MasterWatcher.Shutdown => { cluster.unsubscribe(self) cluster.leave(cluster.selfAddress) context.stop(self) system.scheduler.scheduleOnce(Duration.Zero) { try { - system.awaitTermination(Duration(3, TimeUnit.SECONDS)) + Await.result(system.whenTerminated, Duration(3, TimeUnit.SECONDS)) } catch { - case ex : Exception => //ignore + case ex: Exception => // Ignore } - system.shutdown() + system.terminate() } } } diff --git a/daemon/src/main/scala/io/gearpump/cluster/main/Replay.scala b/daemon/src/main/scala/io/gearpump/cluster/main/Replay.scala index 451376e5c..c9a6e9cc2 100644 --- a/daemon/src/main/scala/io/gearpump/cluster/main/Replay.scala +++ b/daemon/src/main/scala/io/gearpump/cluster/main/Replay.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,12 +17,12 @@ */ package io.gearpump.cluster.main -import io.gearpump.cluster.client.ClientContext -import io.gearpump.util.{AkkaApp, LogUtil} import org.slf4j.Logger -import scala.util.Try +import io.gearpump.cluster.client.ClientContext +import io.gearpump.util.{AkkaApp, LogUtil} +// Internal tool to restart an application object Replay extends AkkaApp with ArgumentsParser { private val LOG: Logger = LogUtil.getLogger(getClass) @@ -31,19 +31,18 @@ object Replay extends AkkaApp with ArgumentsParser { "appid" -> CLIOption("", required = true), // For document purpose only, OPTION_CONFIG option is not used here. // OPTION_CONFIG is parsed by parent shell command "Gear" transparently. - Gear.OPTION_CONFIG -> CLIOption("custom configuration file", required = false, defaultValue = None)) + Gear.OPTION_CONFIG -> CLIOption("custom configuration file", required = false, + defaultValue = None)) override val description = "Replay the application from current min clock(low watermark)" def main(akkaConf: Config, args: Array[String]): Unit = { val config = parse(args) - if (null == config) { - return + if (null != config) { + val client = ClientContext(akkaConf) + client.replayFromTimestampWindowTrailingEdge(config.getInt("appid")) + client.close() } - - val client = ClientContext(akkaConf) - client.replayFromTimestampWindowTrailingEdge(config.getInt("appid")) - client.close() } } diff --git a/daemon/src/main/scala/io/gearpump/cluster/main/Worker.scala b/daemon/src/main/scala/io/gearpump/cluster/main/Worker.scala index 66fe25b16..4818262da 100644 --- a/daemon/src/main/scala/io/gearpump/cluster/main/Worker.scala +++ b/daemon/src/main/scala/io/gearpump/cluster/main/Worker.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,34 +18,38 @@ package io.gearpump.cluster.main +import scala.collection.JavaConverters._ +import scala.concurrent.Await +import scala.concurrent.duration.Duration + import akka.actor.{ActorSystem, Props} +import org.slf4j.Logger + import io.gearpump.cluster.ClusterConfig import io.gearpump.cluster.master.MasterProxy import io.gearpump.cluster.worker.{Worker => WorkerActor} import io.gearpump.transport.HostPort import io.gearpump.util.Constants._ -import io.gearpump.util.{AkkaApp, LogUtil} import io.gearpump.util.LogUtil.ProcessType -import org.slf4j.Logger - -import scala.collection.JavaConverters._ -import scala.util.Try +import io.gearpump.util.{AkkaApp, LogUtil} +/** Tool to start a worker daemon process */ object Worker extends AkkaApp with ArgumentsParser { - override def akkaConfig = ClusterConfig.worker() + protected override def akkaConfig = ClusterConfig.worker() override val description = "Start a worker daemon" - var LOG : Logger = LogUtil.getLogger(getClass) + var LOG: Logger = LogUtil.getLogger(getClass) - def uuid = java.util.UUID.randomUUID.toString + private def uuid = java.util.UUID.randomUUID.toString def main(akkaConf: Config, args: Array[String]): Unit = { val id = uuid this.LOG = { LogUtil.loadConfiguration(akkaConf, ProcessType.WORKER) - //delay creation of LOG instance to avoid creating an empty log file as we reset the log file name here + // Delay creation of LOG instance to avoid creating an empty log file as we + // reset the log file name here LogUtil.getLogger(getClass) } @@ -62,6 +66,6 @@ object Worker extends AkkaApp with ArgumentsParser { system.actorOf(Props(classOf[WorkerActor], masterProxy), classOf[WorkerActor].getSimpleName + id) - system.awaitTermination() + Await.result(system.whenTerminated, Duration.Inf) } } \ No newline at end of file diff --git a/daemon/src/main/scala/io/gearpump/cluster/master/AppManager.scala b/daemon/src/main/scala/io/gearpump/cluster/master/AppManager.scala index e6bd1dbde..058533ef0 100644 --- a/daemon/src/main/scala/io/gearpump/cluster/master/AppManager.scala +++ b/daemon/src/main/scala/io/gearpump/cluster/master/AppManager.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,8 +18,14 @@ package io.gearpump.cluster.master +import scala.concurrent.Future +import scala.concurrent.duration._ +import scala.util.{Failure, Success} + import akka.actor._ import akka.pattern.ask +import org.slf4j.Logger + import io.gearpump.cluster.AppMasterToMaster.{AppDataSaved, SaveAppDataFailed, _} import io.gearpump.cluster.AppMasterToWorker._ import io.gearpump.cluster.ClientToMaster._ @@ -32,32 +38,29 @@ import io.gearpump.cluster.master.InMemoryKVService.{GetKVResult, PutKVResult, P import io.gearpump.cluster.master.Master._ import io.gearpump.util.Constants._ import io.gearpump.util.{ActorUtil, TimeOutScheduler, Util, _} -import org.slf4j.Logger - -import scala.concurrent.Future -import scala.concurrent.duration._ -import scala.util.{Failure, Success} /** - * AppManager is dedicated part of Master to manager all applicaitons. + * AppManager is dedicated child of Master to manager all applications. */ -private[cluster] class AppManager(kvService: ActorRef, launcher: AppMasterLauncherFactory) extends Actor with Stash with TimeOutScheduler{ +private[cluster] class AppManager(kvService: ActorRef, launcher: AppMasterLauncherFactory) + extends Actor with Stash with TimeOutScheduler { + private val LOG: Logger = LogUtil.getLogger(getClass) - private val executorId : Int = APPMASTER_DEFAULT_EXECUTOR_ID + private val executorId: Int = APPMASTER_DEFAULT_EXECUTOR_ID private val appMasterMaxRetries: Int = 5 - private val appMasterRetryTimeRange: Duration = 20 seconds + private val appMasterRetryTimeRange: Duration = 20.seconds implicit val timeout = FUTURE_TIMEOUT implicit val executionContext = context.dispatcher - //next available appId + // Next available appId private var appId: Int = 1 - //from appid to appMaster data + // From appid to appMaster data private var appMasterRegistry = Map.empty[Int, (ActorRef, AppMasterRuntimeInfo)] - // dead appmaster list + // Dead appmaster list private var deadAppMasters = Map.empty[Int, (ActorRef, AppMasterRuntimeInfo)] private var appMasterRestartPolicies = Map.empty[Int, RestartPolicy] @@ -70,7 +73,7 @@ private[cluster] class AppManager(kvService: ActorRef, launcher: AppMasterLaunch def waitForMasterState: Receive = { case GetKVSuccess(_, result) => val masterState = result.asInstanceOf[MasterState] - if(masterState != null) { + if (masterState != null) { this.appId = masterState.maxId + 1 this.deadAppMasters = masterState.deadAppMasters this.appMasterRegistry = masterState.appMasterRegistry @@ -85,11 +88,11 @@ private[cluster] class AppManager(kvService: ActorRef, launcher: AppMasterLaunch stash() } - def receiveHandler : Receive = { + def receiveHandler: Receive = { val msg = "Application Manager started. Ready for application submission..." - System.out.println(msg) LOG.info(msg) - clientMsgHandler orElse appMasterMessage orElse selfMsgHandler orElse workerMessage orElse appDataStoreService orElse terminationWatch + clientMsgHandler orElse appMasterMessage orElse selfMsgHandler orElse workerMessage orElse + appDataStoreService orElse terminationWatch } def clientMsgHandler: Receive = { @@ -97,12 +100,15 @@ private[cluster] class AppManager(kvService: ActorRef, launcher: AppMasterLaunch LOG.info(s"Submit Application ${app.name}($appId) by $username...") val client = sender if (applicationNameExist(app.name)) { - client ! SubmitApplicationResult(Failure(new Exception(s"Application name ${app.name} already existed"))) + client ! SubmitApplicationResult(Failure( + new Exception(s"Application name ${app.name} already existed"))) } else { - context.actorOf(launcher.props(appId, executorId, app, jar, username, context.parent, Some(client)), s"launcher${appId}_${Util.randInt}") + context.actorOf(launcher.props(appId, executorId, app, jar, username, context.parent, + Some(client)), s"launcher${appId}_${Util.randInt()}") val appState = new ApplicationState(appId, app.name, 0, app, jar, username, null) - appMasterRestartPolicies += appId -> new RestartPolicy(appMasterMaxRetries, appMasterRetryTimeRange) + appMasterRestartPolicies += appId -> + new RestartPolicy(appMasterMaxRetries, appMasterRetryTimeRange) kvService ! PutKV(appId.toString, APP_STATE, appState) appId += 1 } @@ -123,7 +129,8 @@ private[cluster] class AppManager(kvService: ActorRef, launcher: AppMasterLaunch } case GetKVFailed(ex) => client ! SubmitApplicationResult(Failure( - new Exception(s"Unable to obtain the Master State. Application $appId will not be restarted.") + new Exception(s"Unable to obtain the Master State. " + + s"Application $appId will not be restarted.") )) } @@ -133,9 +140,11 @@ private[cluster] class AppManager(kvService: ActorRef, launcher: AppMasterLaunch Option(info) match { case Some(info) => val worker = info.worker - LOG.info(s"Shutdown AppMaster at ${Option(worker).map(_.path).orNull}, appId: $appId, executorId: $executorId") + val workerPath = Option(worker).map(_.path).orNull + LOG.info(s"Shutdown AppMaster at ${workerPath}, appId: $appId, executorId: $executorId") cleanApplicationData(appId) - val shutdown = ShutdownExecutor(appId, executorId, s"AppMaster $appId shutdown requested by master...") + val shutdown = ShutdownExecutor(appId, executorId, + s"AppMaster $appId shutdown requested by master...") sendMsgWithTimeOutCallBack(worker, shutdown, 30000, shutDownExecutorTimeOut()) sender ! ShutdownApplicationResult(Success(appId)) case None => @@ -153,18 +162,20 @@ private[cluster] class AppManager(kvService: ActorRef, launcher: AppMasterLaunch case AppMastersDataRequest => var appMastersData = collection.mutable.ListBuffer[AppMasterData]() appMasterRegistry.foreach(pair => { - val (id, (appMaster:ActorRef, info: AppMasterRuntimeInfo)) = pair + val (id, (appMaster: ActorRef, info: AppMasterRuntimeInfo)) = pair val appMasterPath = ActorUtil.getFullPath(context.system, appMaster.path) - val workerPath = Option(info.worker).map(worker => ActorUtil.getFullPath(context.system, worker.path)) + val workerPath = Option(info.worker).map(worker => + ActorUtil.getFullPath(context.system, worker.path)) appMastersData += AppMasterData( AppMasterActive, id, info.appName, appMasterPath, workerPath.orNull, info.submissionTime, info.startTime, info.finishTime, info.user) }) deadAppMasters.foreach(pair => { - val (id, (appMaster:ActorRef, info:AppMasterRuntimeInfo)) = pair + val (id, (appMaster: ActorRef, info: AppMasterRuntimeInfo)) = pair val appMasterPath = ActorUtil.getFullPath(context.system, appMaster.path) - val workerPath = Option(info.worker).map(worker => ActorUtil.getFullPath(context.system, worker.path)) + val workerPath = Option(info.worker).map(worker => + ActorUtil.getFullPath(context.system, worker.path)) appMastersData += AppMasterData( AppMasterInActive, id, info.appName, appMasterPath, workerPath.orNull, @@ -218,7 +229,7 @@ private[cluster] class AppManager(kvService: ActorRef, launcher: AppMasterLaunch case failed: ShutdownExecutorFailed => LOG.error(failed.reason) } - + private def shutDownExecutorTimeOut(): Unit = { LOG.error(s"Shut down executor time out") } @@ -231,7 +242,8 @@ private[cluster] class AppManager(kvService: ActorRef, launcher: AppMasterLaunch LOG.info(s"Register AppMaster for app: ${register.appId} $register") context.watch(appMaster) appMasterRegistry += register.appId -> (appMaster, register) - kvService ! PutKV(MASTER_GROUP, MASTER_STATE, MasterState(appId, appMasterRegistry, deadAppMasters)) + kvService ! PutKV(MASTER_GROUP, MASTER_STATE, + MasterState(appId, appMasterRegistry, deadAppMasters)) sender ! AppMasterRegistered(register.appId) } @@ -257,13 +269,16 @@ private[cluster] class AppManager(kvService: ActorRef, launcher: AppMasterLaunch def terminationWatch: Receive = { case terminate: Terminated => terminate.getAddressTerminated() - LOG.info(s"AppMaster(${terminate.actor.path}) is terminiated, network down: ${terminate.getAddressTerminated()}") - //Now we assume that the only normal way to stop the application is submitting a ShutdownApplication request - val application = appMasterRegistry.find{appInfo => + LOG.info(s"AppMaster(${terminate.actor.path}) is terminiated, " + + s"network down: ${terminate.getAddressTerminated()}") + + // Now we assume that the only normal way to stop the application is submitting a + // ShutdownApplication request + val application = appMasterRegistry.find { appInfo => val (_, (actorRef, _)) = appInfo actorRef.compareTo(terminate.actor) == 0 } - if(application.nonEmpty){ + if (application.nonEmpty) { val appId = application.get._1 (kvService ? GetKV(appId.toString, APP_STATE)).asInstanceOf[Future[GetKVResult]].map { case GetKVSuccess(_, result) => @@ -283,26 +298,29 @@ private[cluster] class AppManager(kvService: ActorRef, launcher: AppMasterLaunch def selfMsgHandler: Receive = { case RecoverApplication(state) => val appId = state.appId - if(appMasterRestartPolicies.get(appId).get.allowRestart) { + if (appMasterRestartPolicies.get(appId).get.allowRestart) { LOG.info(s"AppManager Recovering Application $appId...") - context.actorOf(launcher.props(appId, executorId, state.app, state.jar, state.username, context.parent, None), s"launcher${appId}_${Util.randInt}") + context.actorOf(launcher.props(appId, executorId, state.app, state.jar, state.username, + context.parent, None), s"launcher${appId}_${Util.randInt()}") } else { LOG.error(s"Application $appId failed too many times") } } - case class RecoverApplication(applicationStatus : ApplicationState) + case class RecoverApplication(applicationStatus: ApplicationState) - private def cleanApplicationData(appId : Int) : Unit = { - //add the dead app to dead appMaster + private def cleanApplicationData(appId: Int): Unit = { + // Add the dead app to dead appMaster appMasterRegistry.get(appId).foreach { pair => val (appMasterActor, info) = pair - deadAppMasters += appId -> (appMasterActor, info.copy(finishTime = System.currentTimeMillis())) + deadAppMasters += appId -> (appMasterActor, info.copy( + finishTime = System.currentTimeMillis())) } appMasterRegistry -= appId - kvService ! PutKV(MASTER_GROUP, MASTER_STATE, MasterState(this.appId, appMasterRegistry, deadAppMasters)) + kvService ! PutKV(MASTER_GROUP, MASTER_STATE, + MasterState(this.appId, appMasterRegistry, deadAppMasters)) kvService ! DeleteKVGroup(appId.toString) } @@ -313,7 +331,7 @@ private[cluster] class AppManager(kvService: ActorRef, launcher: AppMasterLaunch object AppManager { final val APP_STATE = "app_state" - //The id is used in KVStore + // The id is used in KVStore final val MASTER_STATE = "master_state" case class MasterState( diff --git a/daemon/src/main/scala/io/gearpump/cluster/master/InMemoryKVService.scala b/daemon/src/main/scala/io/gearpump/cluster/master/InMemoryKVService.scala index fb66a0c4f..616d3eeda 100644 --- a/daemon/src/main/scala/io/gearpump/cluster/master/InMemoryKVService.scala +++ b/daemon/src/main/scala/io/gearpump/cluster/master/InMemoryKVService.scala @@ -19,21 +19,22 @@ package io.gearpump.cluster.master import java.util.concurrent.TimeUnit +import scala.concurrent.TimeoutException +import scala.concurrent.duration.Duration import akka.actor._ import akka.cluster.Cluster -import akka.cluster.ddata.{DistributedData, LWWMap, Key, LWWMapKey} import akka.cluster.ddata.Replicator._ -import io.gearpump.util.{LogUtil} +import akka.cluster.ddata.{DistributedData, LWWMap, LWWMapKey} import org.slf4j.Logger -import scala.concurrent.TimeoutException -import scala.concurrent.duration.Duration + +import io.gearpump.util.LogUtil /** - * A replicated simple in-memory KV service. + * A replicated simple in-memory KV service. The replications are stored on all masters. */ class InMemoryKVService extends Actor with Stash { - import InMemoryKVService._ + import io.gearpump.cluster.master.InMemoryKVService._ private val KV_SERVICE = "gearpump_kvservice" @@ -41,7 +42,7 @@ class InMemoryKVService extends Actor with Stash { private val replicator = DistributedData(context.system).replicator private implicit val cluster = Cluster(context.system) - //optimize write path, we can tolerate one master down for recovery. + // Optimize write path, we can tolerate one master down for recovery. private val timeout = Duration(15, TimeUnit.SECONDS) private val readMajority = ReadMajority(timeout) private val writeMajority = WriteMajority(timeout) @@ -50,39 +51,39 @@ class InMemoryKVService extends Actor with Stash { LWWMapKey[Any](KV_SERVICE + "_" + group) } - def receive : Receive = kvService + def receive: Receive = kvService - def kvService : Receive = { + def kvService: Receive = { - case GetKV(group: String, key : String) => + case GetKV(group: String, key: String) => val request = Request(sender(), key) replicator ! Get(groupKey(group), readMajority, Some(request)) - case success@ GetSuccess(group: LWWMapKey[Any], Some(request: Request)) => + case success@GetSuccess(group: LWWMapKey[Any @unchecked], Some(request: Request)) => val appData = success.get(group) LOG.info(s"Successfully retrived group: ${group.id}") request.client ! GetKVSuccess(request.key, appData.get(request.key).orNull) - case NotFound(group: LWWMapKey[Any], Some(request: Request)) => + case NotFound(group: LWWMapKey[Any @unchecked], Some(request: Request)) => LOG.info(s"We cannot find group $group") request.client ! GetKVSuccess(request.key, null) - case GetFailure(group: LWWMapKey[Any], Some(request: Request)) => + case GetFailure(group: LWWMapKey[Any @unchecked], Some(request: Request)) => val error = s"Failed to get application data, the request key is ${request.key}" LOG.error(error) request.client ! GetKVFailed(new Exception(error)) case PutKV(group: String, key: String, value: Any) => val request = Request(sender(), key) - val update = Update(groupKey(group), LWWMap(), writeMajority, Some(request)) {map => + val update = Update(groupKey(group), LWWMap(), writeMajority, Some(request)) { map => map + (key -> value) } replicator ! update - case UpdateSuccess(group: LWWMapKey[Any], Some(request: Request)) => - request.client ! PutKVSuccess - case ModifyFailure(group: LWWMapKey[Any], error, cause, Some(request: Request)) => + case UpdateSuccess(group: LWWMapKey[Any @unchecked], Some(request: Request)) => + request.client ! PutKVSuccess + case ModifyFailure(group: LWWMapKey[Any @unchecked], error, cause, Some(request: Request)) => request.client ! PutKVFailed(request.key, new Exception(error, cause)) - case UpdateTimeout(group: LWWMapKey[Any], Some(request: Request)) => + case UpdateTimeout(group: LWWMapKey[Any @unchecked], Some(request: Request)) => request.client ! PutKVFailed(request.key, new TimeoutException()) - case delete@ DeleteKVGroup(group: String) => + case delete@DeleteKVGroup(group: String) => replicator ! Delete(groupKey(group), writeMajority) case DeleteSuccess(group) => LOG.info(s"KV Group ${group.id} is deleted") diff --git a/daemon/src/main/scala/io/gearpump/cluster/master/Master.scala b/daemon/src/main/scala/io/gearpump/cluster/master/Master.scala index fdef42e98..0203237ef 100644 --- a/daemon/src/main/scala/io/gearpump/cluster/master/Master.scala +++ b/daemon/src/main/scala/io/gearpump/cluster/master/Master.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,20 +19,26 @@ package io.gearpump.cluster.master import java.lang.management.ManagementFactory +import scala.collection.JavaConverters._ +import scala.collection.immutable import akka.actor._ import akka.remote.DisassociatedEvent import com.typesafe.config.Config +import org.apache.commons.lang.exception.ExceptionUtils +import org.slf4j.Logger + import io.gearpump.cluster.AppMasterToMaster._ import io.gearpump.cluster.ClientToMaster._ import io.gearpump.cluster.ClusterConfig import io.gearpump.cluster.MasterToAppMaster._ -import io.gearpump.cluster.MasterToClient.{HistoryMetricsItem, HistoryMetrics, MasterConfig, ResolveWorkerIdResult} +import io.gearpump.cluster.MasterToClient.{HistoryMetrics, HistoryMetricsItem, MasterConfig, ResolveWorkerIdResult} import io.gearpump.cluster.MasterToWorker._ import io.gearpump.cluster.WorkerToMaster._ import io.gearpump.cluster.master.InMemoryKVService._ import io.gearpump.cluster.master.Master.{MasterInfo, WorkerTerminated, _} import io.gearpump.cluster.scheduler.Scheduler.ApplicationFinished +import io.gearpump.cluster.worker.WorkerId import io.gearpump.jarstore.local.LocalJarStore import io.gearpump.metrics.Metrics.ReportMetrics import io.gearpump.metrics.{JvmMetricsSet, Metrics, MetricsReporterService} @@ -40,31 +46,22 @@ import io.gearpump.transport.HostPort import io.gearpump.util.Constants._ import io.gearpump.util.HistoryMetricsService.HistoryMetricsConfig import io.gearpump.util._ -import io.gearpump.WorkerId - -import org.apache.commons.lang.exception.ExceptionUtils -import org.slf4j.Logger - -import scala.collection.JavaConverters._ -import scala.collection.immutable /** - * Master manage resources of the whole cluster. + * Master Actor who manages resources of the whole cluster. * It is like the resource manager of YARN. - * */ private[cluster] class Master extends Actor with Stash { private val LOG: Logger = LogUtil.getLogger(getClass) - private val systemConfig : Config = context.system.settings.config + private val systemConfig: Config = context.system.settings.config private implicit val timeout = Constants.FUTURE_TIMEOUT private val kvService = context.actorOf(Props(new InMemoryKVService()), "kvService") - // resources and resourceRequests can be dynamically constructed by + // Resources and resourceRequests can be dynamically constructed by // heartbeat of worker and appmaster when master singleton is migrated. - // we don't need to persist them in cluster + // We don't need to persist them in cluster + private var appManager: ActorRef = null - private var appManager : ActorRef = null - - private var scheduler : ActorRef = null + private var scheduler: ActorRef = null private var workers = new immutable.HashMap[ActorRef, WorkerId] @@ -74,16 +71,16 @@ private[cluster] class Master extends Actor with Stash { def receive: Receive = null - // register jvm metrics + // Register jvm metrics Metrics(context.system).register(new JvmMetricsSet(s"master")) LOG.info("master is started at " + ActorUtil.getFullPath(context.system, self.path) + "...") val jarStoreRootPath = systemConfig.getString(Constants.GEARPUMP_APP_JAR_STORE_ROOT_PATH) - val jarStore = if(Util.isLocalPath(jarStoreRootPath)) { + private val jarStore = if (Util.isLocalPath(jarStoreRootPath)) { Some(context.actorOf(Props(classOf[LocalJarStore], jarStoreRootPath))) - } else{ + } else { None } @@ -97,7 +94,8 @@ private[cluster] class Master extends Actor with Stash { context.actorOf(Props(new HistoryMetricsService("master", getHistoryMetricsConfig))) } - val metricsReportService = context.actorOf(Props(new MetricsReporterService(Metrics(context.system)))) + val metricsReportService = context.actorOf( + Props(new MetricsReporterService(Metrics(context.system)))) historyMetricsService.tell(ReportMetrics, metricsReportService) Some(historyMetricsService) } else { @@ -109,7 +107,7 @@ private[cluster] class Master extends Actor with Stash { def waitForNextWorkerId: Receive = { case GetKVSuccess(_, result) => - if(result != null) { + if (result != null) { this.nextWorkerId = result.asInstanceOf[Int] } else { LOG.warn("Cannot find existing state in the distributed cluster...") @@ -124,7 +122,7 @@ private[cluster] class Master extends Actor with Stash { stash() } - def receiveHandler : Receive = workerMsgHandler orElse + def receiveHandler: Receive = workerMsgHandler orElse appMasterMsgHandler orElse clientMsgHandler orElse metricsService orElse @@ -134,7 +132,7 @@ private[cluster] class Master extends Actor with Stash { kvServiceMsgHandler orElse ActorUtil.defaultMsgHandler(self) - def workerMsgHandler : Receive = { + def workerMsgHandler: Receive = { case RegisterNewWorker => val workerId = WorkerId(nextWorkerId, System.currentTimeMillis()) nextWorkerId += 1 @@ -150,41 +148,42 @@ private[cluster] class Master extends Actor with Stash { workers += (sender() -> id) val workerHostname = ActorUtil.getHostname(sender()) LOG.info(s"Register Worker with id $id from $workerHostname ....") - case resourceUpdate : ResourceUpdate => + case resourceUpdate: ResourceUpdate => scheduler forward resourceUpdate } - def jarStoreService : Receive = { + def jarStoreService: Receive = { case GetJarStoreServer => jarStore.foreach(_ forward GetJarStoreServer) } def kvServiceMsgHandler: Receive = { case PutKVSuccess => - //Skip + // Skip case PutKVFailed(key, exception) => - LOG.error(s"Put KV of key $key to InMemoryKVService failed.\n" + ExceptionUtils.getStackTrace(exception)) + LOG.error(s"Put KV of key $key to InMemoryKVService failed.\n" + + ExceptionUtils.getStackTrace(exception)) } - def metricsService : Receive = { + def metricsService: Receive = { case query: QueryHistoryMetrics => if (historyMetricsService.isEmpty) { - // return empty metrics so that we don't hang the UI + // Returns empty metrics so that we don't hang the UI sender ! HistoryMetrics(query.path, List.empty[HistoryMetricsItem]) } else { historyMetricsService.get forward query } } - def appMasterMsgHandler : Receive = { - case request : RequestResource => + def appMasterMsgHandler: Receive = { + case request: RequestResource => scheduler forward request - case registerAppMaster : RegisterAppMaster => - //forward to appManager + case registerAppMaster: RegisterAppMaster => + // Forward to appManager appManager forward registerAppMaster - case save : SaveAppData => + case save: SaveAppData => appManager forward save - case get : GetAppData => + case get: GetAppData => appManager forward get case GetAllWorkers => sender ! WorkerList(workers.values.toList) @@ -219,7 +218,7 @@ private[cluster] class Master extends Actor with Stash { if (cluster.isEmpty) { - //add myself into the list if it is a single node cluster + // Add myself into the list if it is a single node cluster List(hostPort) } else { cluster @@ -228,18 +227,18 @@ private[cluster] class Master extends Actor with Stash { import scala.util.{Failure, Success} - def clientMsgHandler : Receive = { - case app : SubmitApplication => + def clientMsgHandler: Receive = { + case app: SubmitApplication => LOG.debug(s"Receive from client, SubmitApplication $app") appManager.forward(app) - case app : RestartApplication => + case app: RestartApplication => LOG.debug(s"Receive from client, RestartApplication $app") appManager.forward(app) - case app : ShutdownApplication => + case app: ShutdownApplication => LOG.debug(s"Receive from client, Shutting down Application ${app.appId}") scheduler ! ApplicationFinished(app.appId) appManager.forward(app) - case app : ResolveAppId => + case app: ResolveAppId => LOG.debug(s"Receive from client, resolving appId ${app.appId} to ActorRef") appManager.forward(app) case resolve: ResolveWorkerId => @@ -247,7 +246,8 @@ private[cluster] class Master extends Actor with Stash { val worker = workers.find(_._2 == resolve.workerId) worker match { case Some(worker) => sender ! ResolveWorkerIdResult(Success(worker._1)) - case None => sender ! ResolveWorkerIdResult(Failure(new Exception(s"cannot find worker ${resolve.workerId}"))) + case None => sender ! ResolveWorkerIdResult(Failure( + new Exception(s"cannot find worker ${resolve.workerId}"))) } case AppMastersDataRequest => LOG.debug("Master received AppMastersDataRequest") @@ -262,19 +262,20 @@ private[cluster] class Master extends Actor with Stash { sender ! MasterConfig(ClusterConfig.filterOutDefaultConfig(systemConfig)) } - def disassociated : Receive = { - case disassociated : DisassociatedEvent => + def disassociated: Receive = { + case disassociated: DisassociatedEvent => LOG.info(s" disassociated ${disassociated.remoteAddress}") - //LOG.info(s"remote lifecycle events are "+systemConfig.getString("akka.remote.log-remote-lifecycle-events")) } - def terminationWatch : Receive = { - case t : Terminated => + def terminationWatch: Receive = { + case t: Terminated => val actor = t.actor - LOG.info(s"worker ${actor.path} get terminated, is it due to network reason? ${t.getAddressTerminated()}") + LOG.info(s"worker ${actor.path} get terminated, is it due to network reason?" + + t.getAddressTerminated()) + LOG.info("Let's filter out dead resources...") - // filter out dead worker resource - if(workers.keySet.contains(actor)){ + // Filters out dead worker resource + if (workers.keySet.contains(actor)) { scheduler ! WorkerTerminated(workers.get(actor).get) workers -= actor } @@ -283,9 +284,11 @@ private[cluster] class Master extends Actor with Stash { override def preStart(): Unit = { val path = ActorUtil.getFullPath(context.system, self.path) LOG.info(s"master path is $path") - val schedulerClass = Class.forName(systemConfig.getString(Constants.GEARPUMP_SCHEDULING_SCHEDULER)) + val schedulerClass = Class.forName( + systemConfig.getString(Constants.GEARPUMP_SCHEDULING_SCHEDULER)) - appManager = context.actorOf(Props(new AppManager(kvService, AppMasterLauncher)), classOf[AppManager].getSimpleName) + appManager = context.actorOf(Props(new AppManager(kvService, AppMasterLauncher)), + classOf[AppManager].getSimpleName) scheduler = context.actorOf(Props(schedulerClass)) context.system.eventStream.subscribe(self, classOf[DisassociatedEvent]) } @@ -298,10 +301,10 @@ object Master { case class WorkerTerminated(workerId: WorkerId) - case class MasterInfo(master: ActorRef, startTime : Long = 0L) + case class MasterInfo(master: ActorRef, startTime: Long = 0L) object MasterInfo { - def empty = MasterInfo(null) + def empty: MasterInfo = MasterInfo(null) } case class SlotStatus(totalSlots: Int, availableSlots: Int) diff --git a/daemon/src/main/scala/io/gearpump/cluster/scheduler/PriorityScheduler.scala b/daemon/src/main/scala/io/gearpump/cluster/scheduler/PriorityScheduler.scala index 3b1bd9f63..5df008e73 100644 --- a/daemon/src/main/scala/io/gearpump/cluster/scheduler/PriorityScheduler.scala +++ b/daemon/src/main/scala/io/gearpump/cluster/scheduler/PriorityScheduler.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,23 +18,26 @@ package io.gearpump.cluster.scheduler +import scala.collection.mutable + import akka.actor.ActorRef -import io.gearpump.WorkerId + import io.gearpump.cluster.AppMasterToMaster.RequestResource import io.gearpump.cluster.MasterToAppMaster.ResourceAllocated import io.gearpump.cluster.scheduler.Relaxation._ import io.gearpump.cluster.scheduler.Scheduler.PendingRequest +import io.gearpump.cluster.worker.WorkerId -import scala.collection.mutable - +/** Assign resource to application based on the priority of the application */ class PriorityScheduler extends Scheduler { private var resourceRequests = new mutable.PriorityQueue[PendingRequest]()(requestOrdering) - def requestOrdering = new Ordering[PendingRequest] { - override def compare(x: PendingRequest, y: PendingRequest) = { + def requestOrdering: Ordering[PendingRequest] = new Ordering[PendingRequest] { + override def compare(x: PendingRequest, y: PendingRequest): Int = { var res = x.request.priority.id - y.request.priority.id - if (res == 0) + if (res == 0) { res = y.timeStamp.compareTo(x.timeStamp) + } res } } @@ -59,8 +62,10 @@ class PriorityScheduler extends Scheduler { if (newAllocated < request.resource) { val remainingRequest = request.resource - newAllocated val remainingExecutors = request.executorNum - allocations.length - val newResourceRequest = request.copy(resource = remainingRequest, executorNum = remainingExecutors) - scheduleLater = scheduleLater :+ PendingRequest(appId, appMaster, newResourceRequest, timeStamp) + val newResourceRequest = request.copy(resource = remainingRequest, + executorNum = remainingExecutors) + scheduleLater = scheduleLater :+ + PendingRequest(appId, appMaster, newResourceRequest, timeStamp) } allocated = allocated + newAllocated case ONEWORKER => @@ -71,7 +76,8 @@ class PriorityScheduler extends Scheduler { if (availableResource.nonEmpty) { val (workerId, (worker, resource)) = availableResource.get allocated = allocated + request.resource - appMaster ! ResourceAllocated(Array(ResourceAllocation(request.resource, worker, workerId))) + appMaster ! ResourceAllocated(Array(ResourceAllocation(request.resource, worker, + workerId))) resourcesSnapShot.update(workerId, (worker, resource - request.resource)) } else { scheduleLater = scheduleLater :+ PendingRequest(appId, appMaster, request, timeStamp) @@ -80,24 +86,27 @@ class PriorityScheduler extends Scheduler { val workerAndResource = resourcesSnapShot.get(request.workerId) if (workerAndResource.nonEmpty && workerAndResource.get._2 > request.resource) { val (worker, availableResource) = workerAndResource.get - appMaster ! ResourceAllocated(Array(ResourceAllocation(request.resource, worker, request.workerId))) + appMaster ! ResourceAllocated(Array(ResourceAllocation(request.resource, worker, + request.workerId))) allocated = allocated + request.resource - resourcesSnapShot.update(request.workerId, (worker, availableResource - request.resource)) + resourcesSnapShot.update(request.workerId, (worker, + availableResource - request.resource)) } else { scheduleLater = scheduleLater :+ PendingRequest(appId, appMaster, request, timeStamp) } } } - for(request <- scheduleLater) + for (request <- scheduleLater) resourceRequests.enqueue(request) } def resourceRequestHandler: Receive = { case RequestResource(appId, request) => - LOG.info(s"Request resource: appId: $appId, slots: ${request.resource.slots}, relaxation: ${request.relaxation}," + - s" executor number: ${request.executorNum}") + LOG.info(s"Request resource: appId: $appId, slots: ${request.resource.slots}, " + + s"relaxation: ${request.relaxation}, executor number: ${request.executorNum}") val appMaster = sender() - resourceRequests.enqueue(new PendingRequest(appId, appMaster, request, System.currentTimeMillis())) + resourceRequests.enqueue(new PendingRequest(appId, appMaster, request, + System.currentTimeMillis())) allocateResource() } @@ -105,7 +114,9 @@ class PriorityScheduler extends Scheduler { resourceRequests = resourceRequests.filter(_.appId != appId) } - private def allocateFairly(resources: mutable.HashMap[WorkerId, (ActorRef, Resource)], request: ResourceRequest): List[ResourceAllocation] = { + private def allocateFairly( + resources: mutable.HashMap[WorkerId, (ActorRef, Resource)], request: ResourceRequest) + : List[ResourceAllocation] = { val workerNum = resources.size var allocations = List.empty[ResourceAllocation] var totalAvailable = Resource(resources.values.map(_._2.slots).sum) @@ -116,13 +127,16 @@ class PriorityScheduler extends Scheduler { val exeutorNum = Math.min(workerNum, remainingExecutors) val toRequest = Resource(remainingRequest.slots * exeutorNum / remainingExecutors) - val flattenResource = resources.toArray.sortBy(_._2._2.slots)(Ordering[Int].reverse).take(exeutorNum).zipWithIndex.flatMap { workerWithIndex => + val sortedResources = resources.toArray.sortBy(_._2._2.slots)(Ordering[Int].reverse) + val pickedResources = sortedResources.take(exeutorNum) + + val flattenResource = pickedResources.zipWithIndex.flatMap { workerWithIndex => val ((workerId, (worker, resource)), index) = workerWithIndex 0.until(resource.slots).map(seq => ((workerId, worker), seq * workerNum + index)) }.sortBy(_._2).map(_._1) if (flattenResource.length < toRequest.slots) { - //Can not safisfy the user's requirements + // Can not safisfy the user's requirements totalAvailable = Resource.empty } else { flattenResource.take(toRequest.slots).groupBy(actor => actor).mapValues(_.length). diff --git a/daemon/src/main/scala/io/gearpump/cluster/scheduler/Scheduler.scala b/daemon/src/main/scala/io/gearpump/cluster/scheduler/Scheduler.scala index 8ccf1fb8d..ccd105fde 100644 --- a/daemon/src/main/scala/io/gearpump/cluster/scheduler/Scheduler.scala +++ b/daemon/src/main/scala/io/gearpump/cluster/scheduler/Scheduler.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,47 +17,48 @@ */ package io.gearpump.cluster.scheduler +import scala.collection.mutable + import akka.actor.{Actor, ActorRef} -import io.gearpump.cluster.MasterToWorker.UpdateResourceSucceed -import io.gearpump.util.LogUtil -import io.gearpump.{WorkerId, TimeStamp} +import org.slf4j.Logger + +import io.gearpump.TimeStamp import io.gearpump.cluster.MasterToWorker.{UpdateResourceFailed, UpdateResourceSucceed, WorkerRegistered} import io.gearpump.cluster.WorkerToMaster.ResourceUpdate import io.gearpump.cluster.master.Master.WorkerTerminated import io.gearpump.cluster.scheduler.Scheduler.ApplicationFinished +import io.gearpump.cluster.worker.WorkerId import io.gearpump.util.LogUtil -import org.slf4j.Logger - -import scala.collection.mutable /** * Scheduler schedule resource for different applications. */ -abstract class Scheduler extends Actor{ +abstract class Scheduler extends Actor { val LOG: Logger = LogUtil.getLogger(getClass) protected var resources = new mutable.HashMap[WorkerId, (ActorRef, Resource)] - def handleScheduleMessage : Receive = { + def handleScheduleMessage: Receive = { case WorkerRegistered(id, _) => - if(!resources.contains(id)) { + if (!resources.contains(id)) { LOG.info(s"Worker $id added to the scheduler") resources.put(id, (sender, Resource.empty)) } case update@ResourceUpdate(worker, workerId, resource) => LOG.info(s"$update...") - if(resources.contains(workerId)) { + if (resources.contains(workerId)) { val resourceReturned = resource > resources.get(workerId).get._2 resources.update(workerId, (worker, resource)) - if(resourceReturned){ + if (resourceReturned) { allocateResource() } sender ! UpdateResourceSucceed } else { - sender ! UpdateResourceFailed(s"ResourceUpdate failed! The worker $workerId has not been registered into master") + sender ! UpdateResourceFailed( + s"ResourceUpdate failed! The worker $workerId has not been registered into master") } case WorkerTerminated(workerId) => - if(resources.contains(workerId)){ + if (resources.contains(workerId)) { resources -= workerId } case ApplicationFinished(appId) => @@ -69,7 +70,9 @@ abstract class Scheduler extends Actor{ def doneApplication(appId: Int): Unit } -object Scheduler{ - case class PendingRequest(appId: Int, appMaster: ActorRef, request: ResourceRequest, timeStamp: TimeStamp) +object Scheduler { + case class PendingRequest( + appId: Int, appMaster: ActorRef, request: ResourceRequest, timeStamp: TimeStamp) + case class ApplicationFinished(appId: Int) } \ No newline at end of file diff --git a/daemon/src/main/scala/io/gearpump/cluster/worker/DefaultExecutorProcessLauncher.scala b/daemon/src/main/scala/io/gearpump/cluster/worker/DefaultExecutorProcessLauncher.scala index 08a342e35..f97a2090f 100644 --- a/daemon/src/main/scala/io/gearpump/cluster/worker/DefaultExecutorProcessLauncher.scala +++ b/daemon/src/main/scala/io/gearpump/cluster/worker/DefaultExecutorProcessLauncher.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,15 +20,18 @@ package io.gearpump.cluster.worker import java.io.File import com.typesafe.config.Config +import org.slf4j.Logger + import io.gearpump.cluster.scheduler.Resource import io.gearpump.util.{LogUtil, RichProcess, Util} -import org.slf4j.Logger +/** Launcher to start an executor process */ class DefaultExecutorProcessLauncher(val config: Config) extends ExecutorProcessLauncher { private val LOG: Logger = LogUtil.getLogger(getClass) - override def createProcess(appId: Int, executorId: Int, resource: Resource, config: Config, options: Array[String], - classPath: Array[String], mainClass: String, arguments: Array[String]): RichProcess = { + override def createProcess( + appId: Int, executorId: Int, resource: Resource, config: Config, options: Array[String], + classPath: Array[String], mainClass: String, arguments: Array[String]): RichProcess = { LOG.info(s"Launch executor, classpath: ${classPath.mkString(File.pathSeparator)}") Util.startProcess(options, classPath, mainClass, arguments) diff --git a/daemon/src/main/scala/io/gearpump/cluster/worker/Worker.scala b/daemon/src/main/scala/io/gearpump/cluster/worker/Worker.scala index a746b3935..1c22b05a8 100644 --- a/daemon/src/main/scala/io/gearpump/cluster/worker/Worker.scala +++ b/daemon/src/main/scala/io/gearpump/cluster/worker/Worker.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -22,34 +22,33 @@ import java.io.File import java.lang.management.ManagementFactory import java.net.URL import java.util.concurrent.{Executors, TimeUnit} +import scala.concurrent.duration._ +import scala.concurrent.{ExecutionContext, Future, Promise} +import scala.util.{Failure, Success, Try} import akka.actor.SupervisorStrategy.Stop import akka.actor._ import com.typesafe.config.{Config, ConfigFactory} -import io.gearpump.WorkerId +import org.slf4j.Logger + import io.gearpump.cluster.AppMasterToMaster.{GetWorkerData, WorkerData} import io.gearpump.cluster.AppMasterToWorker._ import io.gearpump.cluster.ClientToMaster.{QueryHistoryMetrics, QueryWorkerConfig} -import io.gearpump.cluster.worker.Worker.ExecutorWatcher -import io.gearpump.cluster.{ExecutorJVMConfig, ClusterConfig} import io.gearpump.cluster.MasterToClient.{HistoryMetrics, HistoryMetricsItem, WorkerConfig} import io.gearpump.cluster.MasterToWorker._ import io.gearpump.cluster.WorkerToAppMaster._ import io.gearpump.cluster.WorkerToMaster._ import io.gearpump.cluster.master.Master.MasterInfo import io.gearpump.cluster.scheduler.Resource +import io.gearpump.cluster.worker.Worker.ExecutorWatcher +import io.gearpump.cluster.{ClusterConfig, ExecutorJVMConfig} import io.gearpump.jarstore.JarStoreService import io.gearpump.metrics.Metrics.ReportMetrics import io.gearpump.metrics.{JvmMetricsSet, Metrics, MetricsReporterService} import io.gearpump.util.ActorSystemBooter.Daemon import io.gearpump.util.Constants._ import io.gearpump.util.HistoryMetricsService.HistoryMetricsConfig -import io.gearpump.util.{Constants, TimeOutScheduler, _} -import org.slf4j.Logger - -import scala.concurrent.{Future, Promise, ExecutionContext} -import scala.concurrent.duration._ -import scala.util.{Try, Success, Failure} +import io.gearpump.util.{TimeOutScheduler, _} /** * Worker is used to track the resource on single machine, it is like @@ -57,8 +56,8 @@ import scala.util.{Try, Success, Failure} * * @param masterProxy masterProxy is used to resolve the master */ -private[cluster] class Worker(masterProxy : ActorRef) extends Actor with TimeOutScheduler{ - private val systemConfig : Config = context.system.settings.config +private[cluster] class Worker(masterProxy: ActorRef) extends Actor with TimeOutScheduler { + private val systemConfig: Config = context.system.settings.config private val address = ActorUtil.getFullPath(context.system, self.path) private var resource = Resource.empty @@ -73,14 +72,14 @@ private[cluster] class Worker(masterProxy : ActorRef) extends Actor with TimeOut jarStoreService.init(systemConfig, context.system) private val ioPool = ExecutionContext.fromExecutorService(Executors.newCachedThreadPool()) - private val resourceUpdateTimeoutMs = 30000 //milliseconds + private val resourceUpdateTimeoutMs = 30000 // Milliseconds private var totalSlots: Int = 0 val metricsEnabled = systemConfig.getBoolean(GEARPUMP_METRIC_ENABLED) var historyMetricsService: Option[ActorRef] = None - override def receive : Receive = null + override def receive: Receive = null var LOG: Logger = LogUtil.getLogger(getClass) def service: Receive = @@ -90,10 +89,10 @@ private[cluster] class Worker(masterProxy : ActorRef) extends Actor with TimeOut terminationWatch(masterInfo.master) orElse ActorUtil.defaultMsgHandler(self) - def metricsService : Receive = { + def metricsService: Receive = { case query: QueryHistoryMetrics => if (historyMetricsService.isEmpty) { - // return empty metrics so that we don't hang the UI + // Returns empty metrics so that we don't hang the UI sender ! HistoryMetrics(query.path, List.empty[HistoryMetricsItem]) } else { historyMetricsService.get forward query @@ -104,8 +103,8 @@ private[cluster] class Worker(masterProxy : ActorRef) extends Actor with TimeOut val getHistoryMetricsConfig = HistoryMetricsConfig(systemConfig) - private def initializeMetrics: Unit = { - // register jvm metrics + private def initializeMetrics(): Unit = { + // Registers jvm metrics Metrics(context.system).register(new JvmMetricsSet(s"worker${id}")) historyMetricsService = if (metricsEnabled) { @@ -113,7 +112,8 @@ private[cluster] class Worker(masterProxy : ActorRef) extends Actor with TimeOut context.actorOf(Props(new HistoryMetricsService("worker" + id, getHistoryMetricsConfig))) } - val metricsReportService = context.actorOf(Props(new MetricsReporterService(Metrics(context.system)))) + val metricsReportService = context.actorOf(Props( + new MetricsReporterService(Metrics(context.system)))) historyMetricsService.tell(ReportMetrics, metricsReportService) Some(historyMetricsService) } else { @@ -121,25 +121,27 @@ private[cluster] class Worker(masterProxy : ActorRef) extends Actor with TimeOut } } - def waitForMasterConfirm(killSelf : Cancellable) : Receive = { + def waitForMasterConfirm(timeoutTicker: Cancellable): Receive = { // If master get disconnected, the WorkerRegistered may be triggered multiple times. case WorkerRegistered(id, masterInfo) => this.id = id - // Add the flag check, so that we don't re-initialize when WorkerRegistered - // is triggered multiple times. + // Adds the flag check, so that we don't re-initialize the metrics when worker re-register + // itself. if (!metricsInitialized) { - initializeMetrics + initializeMetrics() metricsInitialized = true } this.masterInfo = masterInfo - killSelf.cancel() + timeoutTicker.cancel() context.watch(masterInfo.master) this.LOG = LogUtil.getLogger(getClass, worker = id) - LOG.info(s"Worker is registered. actor path: ${ActorUtil.getFullPath(context.system, self.path)} ....") - sendMsgWithTimeOutCallBack(masterInfo.master, ResourceUpdate(self, id, resource), resourceUpdateTimeoutMs, updateResourceTimeOut()) + LOG.info(s"Worker is registered. " + + s"actor path: ${ActorUtil.getFullPath(context.system, self.path)} ....") + sendMsgWithTimeOutCallBack(masterInfo.master, ResourceUpdate(self, id, resource), + resourceUpdateTimeoutMs, updateResourceTimeOut()) context.become(service) } @@ -147,31 +149,33 @@ private[cluster] class Worker(masterProxy : ActorRef) extends Actor with TimeOut LOG.error(s"Update worker resource time out") } - def appMasterMsgHandler : Receive = { - case shutdown @ ShutdownExecutor(appId, executorId, reason : String) => + def appMasterMsgHandler: Receive = { + case shutdown@ShutdownExecutor(appId, executorId, reason: String) => val actorName = ActorUtil.actorNameForExecutor(appId, executorId) val executorToStop = executorNameToActor.get(actorName) if (executorToStop.isDefined) { - LOG.info(s"Shutdown executor ${actorName}(${executorToStop.get.path.toString}) due to: $reason") + LOG.info(s"Shutdown executor ${actorName}(${executorToStop.get.path.toString}) " + + s"due to: $reason") executorToStop.get.forward(shutdown) } else { LOG.error(s"Cannot find executor $actorName, ignore this message") sender ! ShutdownExecutorFailed(s"Can not find executor $executorId for app $appId") } - case launch : LaunchExecutor => + case launch: LaunchExecutor => LOG.info(s"$launch") if (resource < launch.resource) { sender ! ExecutorLaunchRejected("There is no free resource on this machine") } else { val actorName = ActorUtil.actorNameForExecutor(launch.appId, launch.executorId) - val executor = context.actorOf(Props(classOf[ExecutorWatcher], launch, masterInfo, ioPool, jarStoreService, executorProcLauncher)) - executorNameToActor += actorName ->executor + val executor = context.actorOf(Props(classOf[ExecutorWatcher], launch, masterInfo, ioPool, + jarStoreService, executorProcLauncher)) + executorNameToActor += actorName -> executor resource = resource - launch.resource allocatedResources = allocatedResources + (executor -> launch.resource) - reportResourceToMaster + reportResourceToMaster() executorsInfo += executor -> ExecutorSlots(launch.appId, launch.executorId, launch.resource.slots) context.watch(executor) @@ -195,26 +199,29 @@ private[cluster] class Worker(masterProxy : ActorRef) extends Actor with TimeOut resource.slots, userDir, jvmName = ManagementFactory.getRuntimeMXBean().getName(), - resourceManagerContainerId = systemConfig.getString(Constants.GEARPUMP_WORKER_RESOURCE_MANAGER_CONTAINER_ID), + resourceManagerContainerId = systemConfig.getString( + GEARPUMP_WORKER_RESOURCE_MANAGER_CONTAINER_ID), historyMetricsConfig = getHistoryMetricsConfig) ) case ChangeExecutorResource(appId, executorId, usedResource) => for (executor <- executorActorRef(appId, executorId); - allocatedResource <- allocatedResources.get(executor)) { + allocatedResource <- allocatedResources.get(executor)) { + allocatedResources += executor -> usedResource resource = resource + allocatedResource - usedResource - reportResourceToMaster + reportResourceToMaster() if (usedResource == Resource(0)) { allocatedResources -= executor // stop executor if there is no resource binded to it. LOG.info(s"Shutdown executor $executorId because the resource used is zero") - executor ! ShutdownExecutor(appId, executorId, "Shutdown executor because the resource used is zero") + executor ! ShutdownExecutor(appId, executorId, + "Shutdown executor because the resource used is zero") } } } - private def reportResourceToMaster: Unit = { + private def reportResourceToMaster(): Unit = { sendMsgWithTimeOutCallBack(masterInfo.master, ResourceUpdate(self, id, resource), resourceUpdateTimeoutMs, updateResourceTimeOut()) } @@ -233,14 +240,28 @@ private[cluster] class Worker(masterProxy : ActorRef) extends Actor with TimeOut } } - def terminationWatch(master : ActorRef) : Receive = { + private def retryRegisterWorker(workerId: WorkerId, timeOutSeconds: Int): Cancellable = { + repeatActionUtil( + seconds = timeOutSeconds, + action = () => { + masterProxy ! RegisterWorker(workerId) + }, + onTimeout = () => { + LOG.error(s"Failed to register the worker $workerId after retrying for $timeOutSeconds " + + s"seconds, abort and kill the worker...") + self ! PoisonPill + }) + } + + def terminationWatch(master: ActorRef): Receive = { case Terminated(actor) => if (actor.compareTo(master) == 0) { - // parent is down, let's make suicide + // Parent master is down, no point to keep worker anymore. Let's make suicide to free + // resources LOG.info(s"Master cannot be contacted, find a new master ...") - context.become(waitForMasterConfirm(repeatActionUtil(30)(masterProxy ! RegisterWorker(id)))) + context.become(waitForMasterConfirm(retryRegisterWorker(id, timeOutSeconds = 30))) } else if (ActorUtil.isChildActorPath(self, actor)) { - //one executor is down, + // One executor is down, LOG.info(s"Executor is down ${getExecutorName(actor)}") val allocated = allocatedResources.get(actor) @@ -248,68 +269,81 @@ private[cluster] class Worker(masterProxy : ActorRef) extends Actor with TimeOut resource = resource + allocated.get executorsInfo -= actor allocatedResources = allocatedResources - actor - sendMsgWithTimeOutCallBack(master, ResourceUpdate(self, id, resource), resourceUpdateTimeoutMs, updateResourceTimeOut()) + sendMsgWithTimeOutCallBack(master, ResourceUpdate(self, id, resource), + resourceUpdateTimeoutMs, updateResourceTimeOut()) } } } - private def getExecutorName(actorRef: ActorRef): Option[String] = { + private def getExecutorName(actorRef: ActorRef): Option[String] = { executorNameToActor.find(_._2 == actorRef).map(_._1) } private def getExecutorProcLauncher(): ExecutorProcessLauncher = { - val launcherClazz = Class.forName(systemConfig.getString(Constants.GEARPUMP_EXECUTOR_PROCESS_LAUNCHER)) - launcherClazz.getConstructor(classOf[Config]).newInstance(systemConfig).asInstanceOf[ExecutorProcessLauncher] + val launcherClazz = Class.forName( + systemConfig.getString(GEARPUMP_EXECUTOR_PROCESS_LAUNCHER)) + launcherClazz.getConstructor(classOf[Config]).newInstance(systemConfig) + .asInstanceOf[ExecutorProcessLauncher] } import context.dispatcher - override def preStart() : Unit = { + override def preStart(): Unit = { LOG.info(s"RegisterNewWorker") - totalSlots = systemConfig.getInt(Constants.GEARPUMP_WORKER_SLOTS) + totalSlots = systemConfig.getInt(GEARPUMP_WORKER_SLOTS) this.resource = Resource(totalSlots) masterProxy ! RegisterNewWorker - context.become(waitForMasterConfirm(repeatActionUtil(30)(Unit))) + context.become(waitForMasterConfirm(registerTimeoutTicker(seconds = 30))) } - private def repeatActionUtil(seconds: Int)(action : => Unit) : Cancellable = { - val cancelSend = context.system.scheduler.schedule(Duration.Zero, Duration(2, TimeUnit.SECONDS))(action) - val cancelSuicide = context.system.scheduler.scheduleOnce(FiniteDuration(seconds, TimeUnit.SECONDS), self, PoisonPill) - return new Cancellable { + private def registerTimeoutTicker(seconds: Int): Cancellable = { + repeatActionUtil(seconds, () => Unit, () => { + LOG.error(s"Failed to register new worker to Master after waiting for $seconds seconds, " + + s"abort and kill the worker...") + self ! PoisonPill + }) + } + + private def repeatActionUtil(seconds: Int, action: () => Unit, onTimeout: () => Unit) + : Cancellable = { + val cancelTimeout = context.system.scheduler.schedule(Duration.Zero, + Duration(2, TimeUnit.SECONDS))(action()) + val cancelSuicide = context.system.scheduler.scheduleOnce(seconds.seconds)(onTimeout()) + new Cancellable { def cancel(): Boolean = { - val result1 = cancelSend.cancel() + val result1 = cancelTimeout.cancel() val result2 = cancelSuicide.cancel() result1 && result2 } def isCancelled: Boolean = { - cancelSend.isCancelled && cancelSuicide.isCancelled + cancelTimeout.isCancelled && cancelSuicide.isCancelled } } } - override def postStop : Unit = { + override def postStop(): Unit = { LOG.info(s"Worker is going down....") ioPool.shutdown() - context.system.shutdown() + context.system.terminate() } } private[cluster] object Worker { - case class ExecutorResult(result : Try[Int]) + case class ExecutorResult(result: Try[Int]) class ExecutorWatcher( - launch: LaunchExecutor, - masterInfo: MasterInfo, - ioPool: ExecutionContext, - jarStoreService: JarStoreService, - procLauncher: ExecutorProcessLauncher) extends Actor { + launch: LaunchExecutor, + masterInfo: MasterInfo, + ioPool: ExecutionContext, + jarStoreService: JarStoreService, + procLauncher: ExecutorProcessLauncher) extends Actor { import launch.{appId, executorId, resource} val executorConfig: Config = { val workerConfig = context.system.settings.config - val submissionConfig = Option(launch.executorJvmConfig).flatMap{ jvmConfig => + val submissionConfig = Option(launch.executorJvmConfig).flatMap { jvmConfig => Option(jvmConfig.executorAkkaConfig) }.getOrElse(ConfigFactory.empty()) @@ -319,15 +353,15 @@ private[cluster] object Worker { // For some config, worker has priority, for others, user Application submission config // have priorities. private def resolveExecutorConfig(workerConfig: Config, submissionConfig: Config): Config = { - val config = submissionConfig.withoutPath(Constants.GEARPUMP_HOSTNAME) - .withoutPath(Constants.GEARPUMP_CLUSTER_MASTERS) - .withoutPath(Constants.GEARPUMP_HOME) - .withoutPath(Constants.GEARPUMP_LOG_DAEMON_DIR) + val config = submissionConfig.withoutPath(GEARPUMP_HOSTNAME) + .withoutPath(GEARPUMP_CLUSTER_MASTERS) + .withoutPath(GEARPUMP_HOME) + .withoutPath(GEARPUMP_LOG_DAEMON_DIR) .withoutPath(GEARPUMP_CLUSTER_EXECUTOR_WORKER_SHARE_SAME_PROCESS) - // fall back to workerConfig + // Falls back to workerConfig .withFallback(workerConfig) - // we should exclude reference.conf, and JVM properties.. + // Excludes reference.conf, and JVM properties.. ClusterConfig.filterOutDefaultConfig(config) } @@ -343,10 +377,10 @@ private[cluster] object Worker { val exitPromise = Promise[Int]() val app = context.actorOf(Props(new InJvmExecutor(launch, exitPromise))) - override def destroy = { + override def destroy(): Unit = { context.stop(app) } - override def exitValue : Future[Int] = { + override def exitValue: Future[Int] = { exitPromise.future } } @@ -376,31 +410,30 @@ private[cluster] object Worker { ctx.classPath.map(path => expandEnviroment(path)) ++ jarPath.map(Array(_)).getOrElse(Array.empty[String]) - - val appLogDir = executorConfig.getString(Constants.GEARPUMP_LOG_APPLICATION_DIR) + val appLogDir = executorConfig.getString(GEARPUMP_LOG_APPLICATION_DIR) val logArgs = List( - s"-D${Constants.GEARPUMP_APPLICATION_ID}=${launch.appId}", - s"-D${Constants.GEARPUMP_EXECUTOR_ID}=${launch.executorId}", - s"-D${Constants.GEARPUMP_MASTER_STARTTIME}=${getFormatedTime(masterInfo.startTime)}", - s"-D${Constants.GEARPUMP_LOG_APPLICATION_DIR}=${appLogDir}") - val configArgs =List(s"-D${Constants.GEARPUMP_CUSTOM_CONFIG_FILE}=$configFile") + s"-D${GEARPUMP_APPLICATION_ID}=${launch.appId}", + s"-D${GEARPUMP_EXECUTOR_ID}=${launch.executorId}", + s"-D${GEARPUMP_MASTER_STARTTIME}=${getFormatedTime(masterInfo.startTime)}", + s"-D${GEARPUMP_LOG_APPLICATION_DIR}=${appLogDir}") + val configArgs = List(s"-D${GEARPUMP_CUSTOM_CONFIG_FILE}=$configFile") - val username = List(s"-D${Constants.GEARPUMP_USERNAME}=${ctx.username}") + val username = List(s"-D${GEARPUMP_USERNAME}=${ctx.username}") - //remote debug executor process - val remoteDebugFlag = executorConfig.getBoolean(Constants.GEARPUMP_REMOTE_DEBUG_EXECUTOR_JVM) + // Remote debug executor process + val remoteDebugFlag = executorConfig.getBoolean(GEARPUMP_REMOTE_DEBUG_EXECUTOR_JVM) val remoteDebugConfig = if (remoteDebugFlag) { - val availablePort = Util.findFreePort.get + val availablePort = Util.findFreePort().get List( "-Xdebug", s"-Xrunjdwp:server=y,transport=dt_socket,address=${availablePort},suspend=n", - s"-D${Constants.GEARPUMP_REMOTE_DEBUG_PORT}=$availablePort" + s"-D${GEARPUMP_REMOTE_DEBUG_PORT}=$availablePort" ) } else { List.empty[String] } - val verboseGCFlag = executorConfig.getBoolean(Constants.GEARPUMP_VERBOSE_GC) + val verboseGCFlag = executorConfig.getBoolean(GEARPUMP_VERBOSE_GC) val verboseGCConfig = if (verboseGCFlag) { List( s"-Xloggc:${appLogDir}/gc-app${launch.appId}-executor-${launch.executorId}.log", @@ -431,7 +464,7 @@ private[cluster] object Worker { var destroyed = false - override def destroy: Unit = { + override def destroy(): Unit = { LOG.info(s"Destroy executor process ${ctx.mainClass}") if (!destroyed) { destroyed = true @@ -449,37 +482,38 @@ private[cluster] object Worker { if (exit == 0) { Future.successful(0) } else { - Future.failed[Int](new Exception(s"Executor exit with failure, exit value: $exit, error summary: ${info.process.logger.error}")) + Future.failed[Int](new Exception(s"Executor exit with failure, exit value: $exit, " + + s"error summary: ${info.process.logger.error}")) } } } } } - import Constants._ private def expandEnviroment(path: String): String = { - //TODO: extend this to support more environment. + // TODO: extend this to support more environment. path.replace(s"<${GEARPUMP_HOME}>", executorConfig.getString(GEARPUMP_HOME)) } - override def preStart: Unit = { - executorHandler.exitValue.onComplete{value => + override def preStart(): Unit = { + executorHandler.exitValue.onComplete { value => procLauncher.cleanProcess(appId, executorId) val result = ExecutorResult(value) self ! result } } - override def postStop: Unit = { - executorHandler.destroy + override def postStop(): Unit = { + executorHandler.destroy() } - //The folders are under ${GEARPUMP_HOME} - val daemonPathPattern = List("lib" + File.separator + "daemon", "lib" + File.separator + "yarn") + // The folders are under ${GEARPUMP_HOME} + val daemonPathPattern = List("lib" + File.separator + "daemon", "lib" + + File.separator + "yarn") override def receive: Receive = { - case ShutdownExecutor(appId, executorId, reason : String) => - executorHandler.destroy + case ShutdownExecutor(appId, executorId, reason: String) => + executorHandler.destroy() sender ! ShutdownExecutorSucceed(appId, executorId) context.stop(self) case ExecutorResult(executorResult) => @@ -506,29 +540,28 @@ private[cluster] object Worker { } trait ExecutorHandler { - def destroy : Unit - def exitValue : Future[Int] + def destroy(): Unit + def exitValue: Future[Int] } case class ProcessInfo(process: RichProcess, jarPath: Option[String], configFile: String) /** - * We will start the executor in the same JVM as worker. - * @param launch - * @param exit - */ - class InJvmExecutor(launch: LaunchExecutor, exit: Promise[Int]) extends Daemon(launch.executorJvmConfig.arguments(0), launch.executorJvmConfig.arguments(1)) { + * Starts the executor in the same JVM as worker. + */ + class InJvmExecutor(launch: LaunchExecutor, exit: Promise[Int]) + extends Daemon(launch.executorJvmConfig.arguments(0), launch.executorJvmConfig.arguments(1)) { private val exitCode = 0 override val supervisorStrategy = - OneForOneStrategy(maxNrOfRetries = 10, withinTimeRange = 1 minute) { + OneForOneStrategy(maxNrOfRetries = 10, withinTimeRange = 1.minute) { case ex: Throwable => LOG.error(s"system $name stopped ", ex) exit.failure(ex) Stop } - override def postStop : Unit = { + override def postStop(): Unit = { if (!exit.isCompleted) { exit.success(exitCode) } diff --git a/daemon/src/main/scala/io/gearpump/jarstore/dfs/DFSJarStoreService.scala b/daemon/src/main/scala/io/gearpump/jarstore/dfs/DFSJarStoreService.scala index 94430ce3b..305bdc152 100644 --- a/daemon/src/main/scala/io/gearpump/jarstore/dfs/DFSJarStoreService.scala +++ b/daemon/src/main/scala/io/gearpump/jarstore/dfs/DFSJarStoreService.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,16 +18,17 @@ package io.gearpump.jarstore.dfs import java.io.File -import akka.actor.{ActorSystem, ActorRefFactory} + +import akka.actor.ActorSystem import com.typesafe.config.Config -import org.apache.hadoop.fs.Path -import io.gearpump.jarstore.{FilePath, JarStoreService} -import io.gearpump.util.LogUtil import org.apache.hadoop.conf.Configuration -import io.gearpump.util.Constants +import org.apache.hadoop.fs.Path import org.apache.hadoop.fs.permission.{FsAction, FsPermission} import org.slf4j.Logger +import io.gearpump.jarstore.{FilePath, JarStoreService} +import io.gearpump.util.{Constants, LogUtil} + /** * DFSJarStoreService store the uploaded jar on HDFS */ @@ -46,7 +47,8 @@ class DFSJarStoreService extends JarStoreService { } /** - * This function will copy the remote file to local file system, called from client side. + * This function will copy the remote file to local file system, called from client side. + * * @param localFile The destination of file path * @param remotePath The remote file path from JarStore */ @@ -60,6 +62,7 @@ class DFSJarStoreService extends JarStoreService { /** * This function will copy the local file to the remote JarStore, called from client side. + * * @param localFile The local file */ override def copyFromLocal(localFile: File): FilePath = { diff --git a/daemon/src/main/scala/io/gearpump/jarstore/local/LocalJarStore.scala b/daemon/src/main/scala/io/gearpump/jarstore/local/LocalJarStore.scala index ec2104bbf..fa1a240a4 100644 --- a/daemon/src/main/scala/io/gearpump/jarstore/local/LocalJarStore.scala +++ b/daemon/src/main/scala/io/gearpump/jarstore/local/LocalJarStore.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,19 +20,17 @@ package io.gearpump.jarstore.local import java.io.File -import akka.actor.{Actor, Props, Stash} -import akka.pattern.{ask, pipe} -import io.gearpump.cluster.ClientToMaster.GetJarStoreServer -import io.gearpump.util._ -import io.gearpump.cluster.ClientToMaster.{JarStoreServerAddress, GetJarStoreServer} +import akka.actor.{Actor, Stash} +import akka.pattern.pipe import org.slf4j.Logger -import scala.concurrent.Future +import io.gearpump.cluster.ClientToMaster.{GetJarStoreServer, JarStoreServerAddress} +import io.gearpump.util._ /** * LocalJarStore store the uploaded jar on local disk. */ -class LocalJarStore(rootDirPath : String) extends Actor with Stash { +class LocalJarStore(rootDirPath: String) extends Actor with Stash { private val LOG: Logger = LogUtil.getLogger(getClass) val host = context.system.settings.config.getString(Constants.GEARPUMP_HOSTNAME) @@ -47,7 +45,7 @@ class LocalJarStore(rootDirPath : String) extends Actor with Stash { server.start pipeTo self - def receive : Receive = { + def receive: Receive = { case FileServer.Port(port) => context.become(listen(port)) unstashAll() @@ -55,7 +53,7 @@ class LocalJarStore(rootDirPath : String) extends Actor with Stash { stash() } - def listen(port : Int) : Receive = { + def listen(port: Int): Receive = { case GetJarStoreServer => sender ! JarStoreServerAddress(s"http://$host:$port/") } diff --git a/daemon/src/main/scala/io/gearpump/jarstore/local/LocalJarStoreService.scala b/daemon/src/main/scala/io/gearpump/jarstore/local/LocalJarStoreService.scala index 9ec7f358d..969ce9080 100644 --- a/daemon/src/main/scala/io/gearpump/jarstore/local/LocalJarStoreService.scala +++ b/daemon/src/main/scala/io/gearpump/jarstore/local/LocalJarStoreService.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,45 +19,48 @@ package io.gearpump.jarstore.local import java.io.File import java.util.concurrent.TimeUnit +import scala.collection.JavaConverters._ +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, ExecutionContext, Future} +import akka.actor.{ActorRef, ActorSystem} import akka.pattern.ask -import akka.actor.{ActorRef, ActorSystem, ActorRefFactory} -import io.gearpump.cluster.ClientToMaster.{JarStoreServerAddress, GetJarStoreServer} -import io.gearpump.cluster.master.MasterProxy import com.typesafe.config.Config -import io.gearpump.jarstore.{FilePath, JarStoreService} -import io.gearpump.util._ -import scala.collection.JavaConversions._ import org.slf4j.Logger -import scala.concurrent.duration.Duration -import scala.concurrent.{ExecutionContext, Await, Future} +import io.gearpump.cluster.ClientToMaster.{GetJarStoreServer, JarStoreServerAddress} +import io.gearpump.cluster.master.MasterProxy +import io.gearpump.jarstore.{FilePath, JarStoreService} +import io.gearpump.util._ /** * LocalJarStoreService store the uploaded jar on local disk. */ -class LocalJarStoreService extends JarStoreService{ +class LocalJarStoreService extends JarStoreService { private def LOG: Logger = LogUtil.getLogger(getClass) private implicit val timeout = Constants.FUTURE_TIMEOUT - private var system : akka.actor.ActorSystem = null - private var master : ActorRef = null + private var system: akka.actor.ActorSystem = null + private var master: ActorRef = null private implicit def dispatcher: ExecutionContext = system.dispatcher override val scheme: String = "file" override def init(config: Config, system: ActorSystem): Unit = { this.system = system - val masters = config.getStringList(Constants.GEARPUMP_CLUSTER_MASTERS).toList.flatMap(Util.parseHostList) - master = system.actorOf(MasterProxy.props(masters), s"masterproxy${Util.randInt}") + val masters = config.getStringList(Constants.GEARPUMP_CLUSTER_MASTERS) + .asScala.flatMap(Util.parseHostList) + master = system.actorOf(MasterProxy.props(masters), s"masterproxy${Util.randInt()}") } - private lazy val client = (master ? GetJarStoreServer).asInstanceOf[Future[JarStoreServerAddress]].map { address => - val client = new FileServer.Client(system, address.url) - client - } + private lazy val client = (master ? GetJarStoreServer).asInstanceOf[Future[JarStoreServerAddress]] + .map { address => + val client = new FileServer.Client(system, address.url) + client + } /** * This function will copy the remote file to local file system, called from client side. + * * @param localFile The destination of file path * @param remotePath The remote file path from JarStore */ diff --git a/daemon/src/main/scala/io/gearpump/util/FileDirective.scala b/daemon/src/main/scala/io/gearpump/util/FileDirective.scala index c39d27eb0..1824a225c 100644 --- a/daemon/src/main/scala/io/gearpump/util/FileDirective.scala +++ b/daemon/src/main/scala/io/gearpump/util/FileDirective.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,8 +18,8 @@ package io.gearpump.util - import java.io.File +import scala.concurrent.{ExecutionContext, Future} import akka.http.scaladsl.model.{HttpEntity, MediaTypes, Multipart} import akka.http.scaladsl.server.Directives._ @@ -28,22 +28,17 @@ import akka.stream.Materializer import akka.stream.scaladsl.FileIO import akka.util.ByteString -import scala.concurrent.{ExecutionContext, Future} - - /** * FileDirective is a set of Akka-http directive to upload/download - * huge binary files. - * + * huge binary files to/from Akka-Http server. */ object FileDirective { - //form field name + // Form field name type Name = String val CHUNK_SIZE = 262144 - /** * File information after a file is uploaded to server. * @@ -71,7 +66,6 @@ object FileDirective { type FormField = Either[FileInfo, String] - /** * directive to uploadFile, it store the uploaded files * to temporary directory, and return a Map from form field name @@ -101,9 +95,7 @@ object FileDirective { } } - /** - * download server file - */ + // Downloads file from server def downloadFile(file: File): Route = { val responseEntity = HttpEntity( MediaTypes.`application/octet-stream`, @@ -112,14 +104,16 @@ object FileDirective { complete(responseEntity) } - private def uploadFileImpl(rootDirectory: File)(implicit mat: Materializer, ec: ExecutionContext): Directive1[Future[Form]] = { + private def uploadFileImpl(rootDirectory: File)(implicit mat: Materializer, ec: ExecutionContext) + : Directive1[Future[Form]] = { Directive[Tuple1[Future[Form]]] { inner => entity(as[Multipart.FormData]) { (formdata: Multipart.FormData) => val form = formdata.parts.mapAsync(1) { p => if (p.filename.isDefined) { - //reserve the suffix - val targetPath = File.createTempFile(s"userfile_${p.name}_", s"${p.filename.getOrElse("")}", rootDirectory) + // Reserve the suffix + val targetPath = File.createTempFile(s"userfile_${p.name}_", + s"${p.filename.getOrElse("")}", rootDirectory) val written = p.entity.dataBytes.runWith(FileIO.toFile(targetPath)) written.map(written => if (written.count > 0) { @@ -128,14 +122,14 @@ object FileDirective { Map.empty[Name, FormField] }) } else { - val valueFuture = p.entity.dataBytes.runFold(ByteString.empty){(total, input) => + val valueFuture = p.entity.dataBytes.runFold(ByteString.empty) {(total, input) => total ++ input } valueFuture.map{value => Map(p.name -> Right(value.utf8String)) } } - }.runFold(new Form(Map.empty[Name, FormField])){(set, value) => + }.runFold(new Form(Map.empty[Name, FormField])) {(set, value) => new Form(set.fields ++ value) } diff --git a/daemon/src/main/scala/io/gearpump/util/FileServer.scala b/daemon/src/main/scala/io/gearpump/util/FileServer.scala index 4be3f2f1d..bf389f78d 100644 --- a/daemon/src/main/scala/io/gearpump/util/FileServer.scala +++ b/daemon/src/main/scala/io/gearpump/util/FileServer.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,28 +17,26 @@ */ package io.gearpump.util - import java.io.File +import scala.concurrent.{ExecutionContext, Future} import akka.actor.ActorSystem import akka.http.scaladsl.Http import akka.http.scaladsl.Http.ServerBinding import akka.http.scaladsl.marshalling.Marshal -import akka.http.scaladsl.model.Uri.{Query, Path} +import akka.http.scaladsl.model.Uri.{Path, Query} import akka.http.scaladsl.model.{HttpEntity, HttpRequest, MediaTypes, Multipart, _} import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server._ import akka.http.scaladsl.unmarshalling.Unmarshal import akka.stream.ActorMaterializer -import akka.http.scaladsl.server.directives.ParameterDirectives.ParamMagnet -import akka.stream.scaladsl.{Sink, Source, FileIO} -import io.gearpump.jarstore.FilePath -import io.gearpump.util.FileDirective._ -import io.gearpump.util.FileServer.Port +import akka.stream.scaladsl.{FileIO, Sink, Source} import spray.json.DefaultJsonProtocol._ import spray.json.JsonFormat -import scala.concurrent.{ExecutionContext, Future} +import io.gearpump.jarstore.FilePath +import io.gearpump.util.FileDirective._ +import io.gearpump.util.FileServer.Port /** * A simple file server implemented with akka-http to store/fetch large @@ -53,7 +51,7 @@ class FileServer(system: ActorSystem, host: String, port: Int = 0, rootDirectory val route: Route = { path("upload") { uploadFileTo(rootDirectory) { form => - val fileName = form.fields.headOption.flatMap{pair => + val fileName = form.fields.headOption.flatMap { pair => val (_, fileInfo) = pair fileInfo match { case Left(file) => Option(file.file).map(_.getName) @@ -90,7 +88,7 @@ class FileServer(system: ActorSystem, host: String, port: Int = 0, rootDirectory """.stripMargin) complete(entity) } - } + } } private var connection: Future[ServerBinding] = null @@ -109,10 +107,10 @@ object FileServer { implicit def filePathFormat: JsonFormat[FilePath] = jsonFormat1(FilePath.apply) - case class Port(port : Int) + case class Port(port: Int) /** - * Client of [[FileServer]] + * Client of [[io.gearpump.util.FileServer]] */ class Client(system: ActorSystem, host: String, port: Int) { @@ -125,27 +123,28 @@ object FileServer { private implicit val ec = system.dispatcher val server = Uri(s"http://$host:$port") - val httpClient = Http(system).outgoingConnection(server.authority.host.address(), server.authority.port) + val httpClient = Http(system).outgoingConnection(server.authority.host.address(), + server.authority.port) def upload(file: File): Future[FilePath] = { val target = server.withPath(Path("/upload")) - val request = entity(file).map{entity => + val request = entity(file).map { entity => HttpRequest(HttpMethods.POST, uri = target, entity = entity) } val response = Source.fromFuture(request).via(httpClient).runWith(Sink.head) - response.flatMap{some => + response.flatMap { some => Unmarshal(some).to[String] - }.map{path => + }.map { path => FilePath(path) } } def download(remoteFile: FilePath, saveAs: File): Future[Unit] = { - val downoad = server.withPath(Path("/download")).withQuery(Query("file" -> remoteFile.path)) - //download file to local - val response = Source.single(HttpRequest(uri = downoad)).via(httpClient).runWith(Sink.head) + val download = server.withPath(Path("/download")).withQuery(Query("file" -> remoteFile.path)) + // Download file to local + val response = Source.single(HttpRequest(uri = download)).via(httpClient).runWith(Sink.head) val downloaded = response.flatMap { response => response.entity.dataBytes.runWith(FileIO.toFile(saveAs)) } @@ -153,7 +152,8 @@ object FileServer { } private def entity(file: File)(implicit ec: ExecutionContext): Future[RequestEntity] = { - val entity = HttpEntity(MediaTypes.`application/octet-stream`, file.length(), FileIO.fromFile(file, chunkSize = 100000)) + val entity = HttpEntity(MediaTypes.`application/octet-stream`, file.length(), + FileIO.fromFile(file, chunkSize = 100000)) val body = Source.single( Multipart.FormData.BodyPart( "uploadfile", diff --git a/daemon/src/test/resources/META-INF/services/io.gearpump.jarstore.JarStoreService b/daemon/src/test/resources/META-INF/services/io.gearpump.jarstore.JarStoreService index f0e0c5c90..d226af9cf 100644 --- a/daemon/src/test/resources/META-INF/services/io.gearpump.jarstore.JarStoreService +++ b/daemon/src/test/resources/META-INF/services/io.gearpump.jarstore.JarStoreService @@ -1,2 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + io.gearpump.jarstore.local.LocalJarStoreService io.gearpump.jarstore.dfs.DFSJarStoreService \ No newline at end of file diff --git a/daemon/src/test/scala/io/gearpump/cluster/MiniCluster.scala b/daemon/src/test/scala/io/gearpump/cluster/MiniCluster.scala index 36f70ed08..c6dbbfe00 100644 --- a/daemon/src/test/scala/io/gearpump/cluster/MiniCluster.scala +++ b/daemon/src/test/scala/io/gearpump/cluster/MiniCluster.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,18 +17,20 @@ */ package io.gearpump.cluster +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, Future} + +import akka.actor.{Actor, ActorRef, ActorSystem, Props} import akka.pattern.ask -import akka.actor.{Actor, ActorRef, Props, ActorSystem} import akka.testkit.TestActorRef import com.typesafe.config.ConfigValueFactory + import io.gearpump.cluster.AppMasterToMaster.GetAllWorkers import io.gearpump.cluster.MasterToAppMaster.WorkerList import io.gearpump.cluster.master.Master import io.gearpump.cluster.worker.Worker import io.gearpump.util.Constants -import scala.concurrent.{Await, Future} - class MiniCluster { private val mockMasterIP = "127.0.0.1" @@ -39,7 +41,7 @@ class MiniCluster { val master = system.actorOf(Props(classOf[Master]), "master") val worker = system.actorOf(Props(classOf[Worker], master), "worker") - //wait until worker register itself to master + // Wait until worker register itself to master waitUtilWorkerIsRegistered(master) (master, worker) } @@ -48,12 +50,10 @@ class MiniCluster { TestActorRef(props) } - private def waitUtilWorkerIsRegistered(master: ActorRef): Unit = { - while(!isWorkerRegistered(master)) {} + while (!isWorkerRegistered(master)) {} } - private def isWorkerRegistered(master: ActorRef): Boolean = { import scala.concurrent.duration._ implicit val dispatcher = system.dispatcher @@ -62,13 +62,13 @@ class MiniCluster { val workerListFuture = (master ? GetAllWorkers).asInstanceOf[Future[WorkerList]] - // wait until the worker is registered. - val workers = Await.result[WorkerList](workerListFuture, 15 seconds) + // Waits until the worker is registered. + val workers = Await.result[WorkerList](workerListFuture, 15.seconds) workers.workers.size > 0 } - def shutDown() = { - system.shutdown() - system.awaitTermination() + def shutDown(): Unit = { + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } } diff --git a/daemon/src/test/scala/io/gearpump/cluster/main/MainSpec.scala b/daemon/src/test/scala/io/gearpump/cluster/main/MainSpec.scala index 9dc028953..30347d2ae 100644 --- a/daemon/src/test/scala/io/gearpump/cluster/main/MainSpec.scala +++ b/daemon/src/test/scala/io/gearpump/cluster/main/MainSpec.scala @@ -15,34 +15,34 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package io.gearpump.cluster.main -import io.gearpump.cluster.MasterToAppMaster.{ReplayFromTimestampWindowTrailingEdge, AppMastersDataRequest} +import scala.concurrent.Future +import scala.util.{Success, Try} + +import com.typesafe.config.Config +import org.scalatest._ + import io.gearpump.cluster.ClientToMaster.{ResolveAppId, ShutdownApplication} -import io.gearpump.cluster.MasterToAppMaster._ -import io.gearpump.cluster.MasterToClient.{ResolveAppIdResult, ReplayApplicationResult, ShutdownApplicationResult} +import io.gearpump.cluster.MasterToAppMaster.{AppMastersDataRequest, ReplayFromTimestampWindowTrailingEdge, _} +import io.gearpump.cluster.MasterToClient.{ReplayApplicationResult, ResolveAppIdResult, ShutdownApplicationResult} import io.gearpump.cluster.WorkerToMaster.RegisterNewWorker import io.gearpump.cluster.{MasterHarness, TestUtil} import io.gearpump.util.Constants._ import io.gearpump.util.{Constants, LogUtil, Util} -import org.scalatest._ - -import scala.concurrent.duration.Duration -import scala.util.{Success, Try} - -import scala.concurrent.Future class MainSpec extends FlatSpec with Matchers with BeforeAndAfterEach with MasterHarness { private val LOG = LogUtil.getLogger(getClass) - override def config = TestUtil.DEFAULT_CONFIG + override def config: Config = TestUtil.DEFAULT_CONFIG - override def beforeEach() = { + override def beforeEach(): Unit = { startActorSystem() } - override def afterEach() = { + override def afterEach(): Unit = { shutdownActorSystem() } @@ -62,7 +62,6 @@ class MainSpec extends FlatSpec with Matchers with BeforeAndAfterEach with Maste getMainClassName(Worker), Array.empty) - try { masterReceiver.expectMsg(PROCESS_BOOT_TIME, RegisterNewWorker) @@ -72,38 +71,41 @@ class MainSpec extends FlatSpec with Matchers with BeforeAndAfterEach with Maste } } -// This UT fails a lot on Travis, temporarily delete it. -// "Master" should "accept worker RegisterNewWorker when started" in { -// val worker = TestProbe()(getActorSystem) -// -// val port = Util.findFreePort.get -// -// val masterConfig = Array(s"-D${Constants.GEARPUMP_CLUSTER_MASTERS}.0=127.0.0.1:$port", -// s"-D${Constants.GEARPUMP_HOSTNAME}=127.0.0.1") -// -// val masterProcess = Util.startProcess(masterConfig, -// getContextClassPath, -// getMainClassName(io.gearpump.cluster.main.Master), -// Array("-ip", "127.0.0.1", "-port", port.toString)) -// -// //wait for master process to be started -// -// try { -// -// val masterProxy = getActorSystem.actorOf(MasterProxy.props(List(HostPort("127.0.0.1", port))), "mainSpec") -// -// worker.send(masterProxy, RegisterNewWorker) -// worker.expectMsgType[WorkerRegistered](PROCESS_BOOT_TIME) -// } finally { -// masterProcess.destroy() -// } -// } + // This UT fails a lot on Travis, temporarily delete it. + // "Master" should "accept worker RegisterNewWorker when started" in { + // val worker = TestProbe()(getActorSystem) + // + // val port = Util.findFreePort.get + // + // val masterConfig = Array(s"-D${Constants.GEARPUMP_CLUSTER_MASTERS}.0=127.0.0.1:$port", + // s"-D${Constants.GEARPUMP_HOSTNAME}=127.0.0.1") + // + // val masterProcess = Util.startProcess(masterConfig, + // getContextClassPath, + // getMainClassName(io.gearpump.cluster.main.Master), + // Array("-ip", "127.0.0.1", "-port", port.toString)) + // + // //wait for master process to be started + // + // try { + // + // val masterProxy = getActorSystem.actorOf( + // MasterProxy.props(List(HostPort("127.0.0.1", port))), "mainSpec") + // + // worker.send(masterProxy, RegisterNewWorker) + // worker.expectMsgType[WorkerRegistered](PROCESS_BOOT_TIME) + // } finally { + // masterProcess.destroy() + // } + // } "Info" should "be started without exception" in { val masterReceiver = createMockMaster() - Future {io.gearpump.cluster.main.Info.main(masterConfig, Array.empty)} + Future { + io.gearpump.cluster.main.Info.main(masterConfig, Array.empty) + } masterReceiver.expectMsg(PROCESS_BOOT_TIME, AppMastersDataRequest) masterReceiver.reply(AppMastersData(List(AppMasterData(AppMasterActive, 0, "appName")))) @@ -113,7 +115,9 @@ class MainSpec extends FlatSpec with Matchers with BeforeAndAfterEach with Maste val masterReceiver = createMockMaster() - Future {Kill.main(masterConfig, Array("-appid", "0"))} + Future { + Kill.main(masterConfig, Array("-appid", "0")) + } masterReceiver.expectMsg(PROCESS_BOOT_TIME, ShutdownApplication(0)) masterReceiver.reply(ShutdownApplicationResult(Success(0))) @@ -123,7 +127,9 @@ class MainSpec extends FlatSpec with Matchers with BeforeAndAfterEach with Maste val masterReceiver = createMockMaster() - Future {Replay.main(masterConfig, Array("-appid", "0"))} + Future { + Replay.main(masterConfig, Array("-appid", "0")) + } masterReceiver.expectMsgType[ResolveAppId](PROCESS_BOOT_TIME) masterReceiver.reply(ResolveAppIdResult(Success(masterReceiver.ref))) @@ -132,7 +138,7 @@ class MainSpec extends FlatSpec with Matchers with BeforeAndAfterEach with Maste } "Local" should "be started without exception" in { - val port = Util.findFreePort.get + val port = Util.findFreePort().get val options = Array(s"-D${Constants.GEARPUMP_CLUSTER_MASTERS}.0=$getHost:$port", s"-D${Constants.GEARPUMP_HOSTNAME}=$getHost", s"-D${PREFER_IPV4}=true") @@ -149,14 +155,15 @@ class MainSpec extends FlatSpec with Matchers with BeforeAndAfterEach with Maste val result = fn if (result || times <= 0) { result - } else { + } else { Thread.sleep(1000) retry(times - 1)(fn) } } try { - assert(retry(10)(isPortUsed("127.0.0.1", port)), "local is not started successfully, as port is not used " + port) + assert(retry(10)(isPortUsed("127.0.0.1", port)), + "local is not started successfully, as port is not used " + port) } finally { local.destroy() } @@ -169,9 +176,8 @@ class MainSpec extends FlatSpec with Matchers with BeforeAndAfterEach with Maste assert(Try(Gear.main(Array.empty)).isSuccess, "print help, no throw") for (command <- commands) { - //Temporarily disable this test - //assert(Try(Gear.main(Array(command))).isSuccess, "print help, no throw, command: " + command) - assert(Try(Gear.main(Array("-noexist"))).isFailure, "pass unknown option, throw, command: " + command) + assert(Try(Gear.main(Array("-noexist"))).isFailure, + "pass unknown option, throw, command: " + command) } assert(Try(Gear.main(Array("unknownCommand"))).isFailure, "unknown command, throw ") diff --git a/daemon/src/test/scala/io/gearpump/cluster/main/MasterWatcherSpec.scala b/daemon/src/test/scala/io/gearpump/cluster/main/MasterWatcherSpec.scala index 39279939f..66b9ea896 100644 --- a/daemon/src/test/scala/io/gearpump/cluster/main/MasterWatcherSpec.scala +++ b/daemon/src/test/scala/io/gearpump/cluster/main/MasterWatcherSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,15 +17,15 @@ */ package io.gearpump.cluster.main +import scala.concurrent.Await +import scala.concurrent.duration._ + import akka.actor.{ActorSystem, Props} -import akka.testkit.{TestActorRef, TestProbe} +import akka.testkit.TestProbe import com.typesafe.config.Config -import io.gearpump.cluster.{TestUtil, MasterHarness} -import io.gearpump.cluster.{MasterHarness, TestUtil} -import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} +import org.scalatest.{FlatSpec, Matchers} -import scala.concurrent.duration._ -import scala.language.postfixOps +import io.gearpump.cluster.TestUtil class MasterWatcherSpec extends FlatSpec with Matchers { def config: Config = TestUtil.MASTER_CONFIG @@ -37,8 +37,8 @@ class MasterWatcherSpec extends FlatSpec with Matchers { val masterWatcher = system.actorOf(Props(classOf[MasterWatcher], "watcher")) actorWatcher watch masterWatcher - actorWatcher.expectTerminated(masterWatcher, 5 seconds) - system.shutdown() - system.awaitTermination() + actorWatcher.expectTerminated(masterWatcher, 5.seconds) + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } } diff --git a/daemon/src/test/scala/io/gearpump/cluster/master/AppManagerSpec.scala b/daemon/src/test/scala/io/gearpump/cluster/master/AppManagerSpec.scala index ad19675c8..ee6e0e2aa 100644 --- a/daemon/src/test/scala/io/gearpump/cluster/master/AppManagerSpec.scala +++ b/daemon/src/test/scala/io/gearpump/cluster/master/AppManagerSpec.scala @@ -18,42 +18,44 @@ package io.gearpump.cluster.master +import scala.util.Success + import akka.actor.{Actor, ActorRef, Props} import akka.testkit.TestProbe -import io.gearpump.cluster.AppMasterToMaster.AppDataSaved -import io.gearpump.cluster.MasterToAppMaster.{AppMasterData, AppMastersData, AppMastersDataRequest, AppMasterRegistered} -import io.gearpump.cluster.TestUtil -import io.gearpump.cluster.AppMasterToMaster._ +import com.typesafe.config.Config +import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} + +import io.gearpump.cluster.AppMasterToMaster.{AppDataSaved, _} import io.gearpump.cluster.ClientToMaster.{ResolveAppId, ShutdownApplication, SubmitApplication} -import io.gearpump.cluster.MasterToAppMaster._ -import io.gearpump.cluster.MasterToClient.{SubmitApplicationResult, ShutdownApplicationResult, ReplayApplicationResult, ResolveAppIdResult} -import io.gearpump.cluster._ -import io.gearpump.cluster.master.AppManager._ +import io.gearpump.cluster.MasterToAppMaster.{AppMasterData, AppMasterRegistered, AppMastersData, AppMastersDataRequest, _} +import io.gearpump.cluster.MasterToClient.{ResolveAppIdResult, ShutdownApplicationResult, SubmitApplicationResult} import io.gearpump.cluster.appmaster.{AppMasterRuntimeInfo, ApplicationState} +import io.gearpump.cluster.master.AppManager._ import io.gearpump.cluster.master.InMemoryKVService.{GetKV, GetKVSuccess, PutKV, PutKVSuccess} -import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} - -import scala.util.Success +import io.gearpump.cluster.{TestUtil, _} +import io.gearpump.util.LogUtil class AppManagerSpec extends FlatSpec with Matchers with BeforeAndAfterEach with MasterHarness { var kvService: TestProbe = null var haService: TestProbe = null var appLauncher: TestProbe = null - var appManager : ActorRef = null + var appManager: ActorRef = null + private val LOG = LogUtil.getLogger(getClass) - override def config = TestUtil.DEFAULT_CONFIG + override def config: Config = TestUtil.DEFAULT_CONFIG - override def beforeEach() = { + override def beforeEach(): Unit = { startActorSystem() kvService = TestProbe()(getActorSystem) appLauncher = TestProbe()(getActorSystem) - appManager = getActorSystem.actorOf(Props(new AppManager(kvService.ref, new DummyAppMasterLauncherFactory(appLauncher)))) + appManager = getActorSystem.actorOf(Props(new AppManager(kvService.ref, + new DummyAppMasterLauncherFactory(appLauncher)))) kvService.expectMsgType[GetKV] kvService.reply(GetKVSuccess(MASTER_STATE, MasterState(0, Map.empty, Map.empty))) } - override def afterEach() = { + override def afterEach(): Unit = { shutdownActorSystem() } @@ -84,7 +86,7 @@ class AppManagerSpec extends FlatSpec with Matchers with BeforeAndAfterEach with } "AppManager" should "support application submission and recover if appmaster dies" in { - Console.out.println("=================testing recover==============") + LOG.info("=================testing recover==============") testClientSubmission(withRecover = true) } @@ -112,14 +114,15 @@ class AppManagerSpec extends FlatSpec with Matchers with BeforeAndAfterEach with kvService.expectMsgType[PutKV] appLauncher.expectMsg(LauncherStarted(appId)) - appMaster.send(appManager, RegisterAppMaster(appMaster.ref, AppMasterRuntimeInfo(appId, app.name))) + appMaster.send(appManager, RegisterAppMaster(appMaster.ref, + AppMasterRuntimeInfo(appId, app.name))) appMaster.expectMsgType[AppMasterRegistered] client.send(appManager, submit) assert(client.receiveN(1).head.asInstanceOf[SubmitApplicationResult].appId.isFailure) } - def testClientSubmission(withRecover: Boolean) : Unit = { + def testClientSubmission(withRecover: Boolean): Unit = { val app = TestUtil.dummyApp val submit = SubmitApplication(app, None, "username") val client = TestProbe()(getActorSystem) @@ -131,7 +134,8 @@ class AppManagerSpec extends FlatSpec with Matchers with BeforeAndAfterEach with kvService.expectMsgType[PutKV] appLauncher.expectMsg(LauncherStarted(appId)) - appMaster.send(appManager, RegisterAppMaster(appMaster.ref, AppMasterRuntimeInfo(appId, app.name))) + appMaster.send(appManager, RegisterAppMaster(appMaster.ref, + AppMasterRuntimeInfo(appId, app.name))) kvService.expectMsgType[PutKV] appMaster.expectMsgType[AppMasterRegistered] @@ -148,10 +152,10 @@ class AppManagerSpec extends FlatSpec with Matchers with BeforeAndAfterEach with client.send(appManager, ShutdownApplication(appId)) client.expectMsg(ShutdownApplicationResult(Success(appId))) } else { - //do recover + // Do recovery getActorSystem.stop(appMaster.ref) kvService.expectMsgType[GetKV] - val appState = ApplicationState(appId, "application1", 1, app , None, "username", null) + val appState = ApplicationState(appId, "application1", 1, app, None, "username", null) kvService.reply(GetKVSuccess(APP_STATE, appState)) appLauncher.expectMsg(LauncherStarted(appId)) } @@ -160,7 +164,8 @@ class AppManagerSpec extends FlatSpec with Matchers with BeforeAndAfterEach with class DummyAppMasterLauncherFactory(test: TestProbe) extends AppMasterLauncherFactory { - override def props(appId: Int, executorId: Int, app: AppDescription, jar: Option[AppJar], username: String, master: ActorRef, client: Option[ActorRef]): Props = { + override def props(appId: Int, executorId: Int, app: AppDescription, jar: Option[AppJar], + username: String, master: ActorRef, client: Option[ActorRef]): Props = { Props(new DummyAppMasterLauncher(test, appId)) } } @@ -169,8 +174,8 @@ class DummyAppMasterLauncher(test: TestProbe, appId: Int) extends Actor { test.ref ! LauncherStarted(appId) override def receive: Receive = { - case any : Any => test.ref forward any + case any: Any => test.ref forward any } } -case class LauncherStarted(appId : Int) +case class LauncherStarted(appId: Int) diff --git a/daemon/src/test/scala/io/gearpump/cluster/master/InMemoryKVServiceSpec.scala b/daemon/src/test/scala/io/gearpump/cluster/master/InMemoryKVServiceSpec.scala index 8f60d340a..b9293495c 100644 --- a/daemon/src/test/scala/io/gearpump/cluster/master/InMemoryKVServiceSpec.scala +++ b/daemon/src/test/scala/io/gearpump/cluster/master/InMemoryKVServiceSpec.scala @@ -18,28 +18,32 @@ package io.gearpump.cluster.master +import scala.concurrent.duration._ + import akka.actor.Props import akka.testkit.TestProbe +import com.typesafe.config.Config +import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} + import io.gearpump.cluster.master.InMemoryKVService._ import io.gearpump.cluster.{MasterHarness, TestUtil} -import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} -import scala.concurrent.duration._ -class InMemoryKVServiceSpec extends FlatSpec with Matchers with BeforeAndAfterEach with MasterHarness { +class InMemoryKVServiceSpec + extends FlatSpec with Matchers with BeforeAndAfterEach with MasterHarness { - override def beforeEach() = { + override def beforeEach(): Unit = { startActorSystem() } - override def afterEach() = { + override def afterEach(): Unit = { shutdownActorSystem() } - override def config = TestUtil.MASTER_CONFIG + override def config: Config = TestUtil.MASTER_CONFIG "KVService" should "get, put, delete correctly" in { - val system = getActorSystem - val kvService = system.actorOf(Props(new InMemoryKVService())) + val system = getActorSystem + val kvService = system.actorOf(Props(new InMemoryKVService())) val group = "group" val client = TestProbe()(system) @@ -55,11 +59,11 @@ class InMemoryKVServiceSpec extends FlatSpec with Matchers with BeforeAndAfterEa client.send(kvService, DeleteKVGroup(group)) - // after DeleteGroup, it no longer accept Get and Put + // After DeleteGroup, it no longer accept Get and Put message for this group. client.send(kvService, GetKV(group, "key")) - client.expectNoMsg(3 seconds) + client.expectNoMsg(3.seconds) client.send(kvService, PutKV(group, "key", 3)) - client.expectNoMsg(3 seconds) + client.expectNoMsg(3.seconds) } } diff --git a/daemon/src/test/scala/io/gearpump/cluster/scheduler/PrioritySchedulerSpec.scala b/daemon/src/test/scala/io/gearpump/cluster/scheduler/PrioritySchedulerSpec.scala index aa2028b89..a75ade2bd 100644 --- a/daemon/src/test/scala/io/gearpump/cluster/scheduler/PrioritySchedulerSpec.scala +++ b/daemon/src/test/scala/io/gearpump/cluster/scheduler/PrioritySchedulerSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,25 +17,26 @@ */ package io.gearpump.cluster.scheduler +import scala.concurrent.duration._ + import akka.actor.{ActorSystem, Props} import akka.testkit.{ImplicitSender, TestKit, TestProbe} -import io.gearpump.WorkerId +import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} + import io.gearpump.cluster.AppMasterToMaster.RequestResource import io.gearpump.cluster.MasterToAppMaster.ResourceAllocated import io.gearpump.cluster.MasterToWorker.{UpdateResourceFailed, WorkerRegistered} import io.gearpump.cluster.TestUtil import io.gearpump.cluster.WorkerToMaster.ResourceUpdate import io.gearpump.cluster.master.Master.MasterInfo +import io.gearpump.cluster.scheduler.Priority.{HIGH, LOW, NORMAL} import io.gearpump.cluster.scheduler.Scheduler.ApplicationFinished -import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} - -import scala.concurrent.duration._ -import scala.language.postfixOps +import io.gearpump.cluster.worker.WorkerId class PrioritySchedulerSpec(_system: ActorSystem) extends TestKit(_system) with ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll{ - def this() = this(ActorSystem("PrioritySchedulerSpec", TestUtil.DEFAULT_CONFIG)) + def this() = this(ActorSystem("PrioritySchedulerSpec", TestUtil.DEFAULT_CONFIG)) val appId = 0 val workerId1: WorkerId = WorkerId(1, 0L) val workerId2: WorkerId = WorkerId(2, 0L) @@ -51,28 +52,33 @@ class PrioritySchedulerSpec(_system: ActorSystem) extends TestKit(_system) with "update resource only when the worker is registered" in { val scheduler = system.actorOf(Props(classOf[PriorityScheduler])) scheduler ! ResourceUpdate(mockWorker1.ref, workerId1, Resource(100)) - expectMsg(UpdateResourceFailed(s"ResourceUpdate failed! The worker $workerId1 has not been registered into master")) + expectMsg(UpdateResourceFailed(s"ResourceUpdate failed! The worker $workerId1 has not been " + + s"registered into master")) } "drop application's resource requests when the application is removed" in { val scheduler = system.actorOf(Props(classOf[PriorityScheduler])) - val request1 = ResourceRequest(Resource(40), WorkerId.unspecified, Priority.HIGH, Relaxation.ANY) - val request2 = ResourceRequest(Resource(20), WorkerId.unspecified, Priority.HIGH, Relaxation.ANY) + val request1 = ResourceRequest(Resource(40), WorkerId.unspecified, HIGH, Relaxation.ANY) + val request2 = ResourceRequest(Resource(20), WorkerId.unspecified, HIGH, Relaxation.ANY) scheduler.tell(RequestResource(appId, request1), mockAppMaster.ref) scheduler.tell(RequestResource(appId, request2), mockAppMaster.ref) scheduler.tell(ApplicationFinished(appId), mockAppMaster.ref) scheduler.tell(WorkerRegistered(workerId1, MasterInfo.empty), mockWorker1.ref) scheduler.tell(ResourceUpdate(mockWorker1.ref, workerId1, Resource(100)), mockWorker1.ref) - mockAppMaster.expectNoMsg(5 seconds) + mockAppMaster.expectNoMsg(5.seconds) } } + def sameElement(left: ResourceAllocated, right: ResourceAllocated): Boolean = { + left.allocations.sortBy(_.workerId).sameElements(right.allocations.sortBy(_.workerId)) + } + "The resource request with higher priority" should { "be handled first" in { val scheduler = system.actorOf(Props(classOf[PriorityScheduler])) - val request1 = ResourceRequest(Resource(40), WorkerId.unspecified, Priority.LOW, Relaxation.ANY) - val request2 = ResourceRequest(Resource(20), WorkerId.unspecified, Priority.NORMAL, Relaxation.ANY) - val request3 = ResourceRequest(Resource(30), WorkerId.unspecified, Priority.HIGH, Relaxation.ANY) + val request1 = ResourceRequest(Resource(40), WorkerId.unspecified, LOW, Relaxation.ANY) + val request2 = ResourceRequest(Resource(20), WorkerId.unspecified, NORMAL, Relaxation.ANY) + val request3 = ResourceRequest(Resource(30), WorkerId.unspecified, HIGH, Relaxation.ANY) scheduler.tell(RequestResource(appId, request1), mockAppMaster.ref) scheduler.tell(RequestResource(appId, request1), mockAppMaster.ref) @@ -81,58 +87,107 @@ class PrioritySchedulerSpec(_system: ActorSystem) extends TestKit(_system) with scheduler.tell(WorkerRegistered(workerId1, MasterInfo.empty), mockWorker1.ref) scheduler.tell(ResourceUpdate(mockWorker1.ref, workerId1, Resource(100)), mockWorker1.ref) - mockAppMaster.expectMsg(5 seconds, ResourceAllocated(Array(ResourceAllocation(Resource(30), mockWorker1.ref, workerId1)))) - mockAppMaster.expectMsg(5 seconds, ResourceAllocated(Array(ResourceAllocation(Resource(20), mockWorker1.ref, workerId1)))) - mockAppMaster.expectMsg(5 seconds, ResourceAllocated(Array(ResourceAllocation(Resource(40), mockWorker1.ref, workerId1)))) + var expect = ResourceAllocated( + Array(ResourceAllocation(Resource(30), mockWorker1.ref, workerId1))) + mockAppMaster.expectMsgPF(5.seconds) { + case request: ResourceAllocated if sameElement(request, expect) => Unit + } + + expect = ResourceAllocated( + Array(ResourceAllocation(Resource(20), mockWorker1.ref, workerId1))) + mockAppMaster.expectMsgPF(5.seconds) { + case request: ResourceAllocated if sameElement(request, expect) => Unit + } + + expect = ResourceAllocated( + Array(ResourceAllocation(Resource(40), mockWorker1.ref, workerId1))) + mockAppMaster.expectMsgPF(5.seconds) { + case request: ResourceAllocated if sameElement(request, expect) => Unit + } scheduler.tell(WorkerRegistered(workerId2, MasterInfo.empty), mockWorker2.ref) scheduler.tell(ResourceUpdate(mockWorker1.ref, workerId1, Resource.empty), mockWorker1.ref) scheduler.tell(ResourceUpdate(mockWorker2.ref, workerId2, Resource(100)), mockWorker2.ref) - mockAppMaster.expectMsg(5 seconds, ResourceAllocated(Array(ResourceAllocation(Resource(40), mockWorker2.ref, workerId2)))) + + expect = ResourceAllocated( + Array(ResourceAllocation(Resource(40), mockWorker2.ref, workerId2))) + mockAppMaster.expectMsgPF(5.seconds) { + case request: ResourceAllocated if sameElement(request, expect) => Unit + } } } "The resource request which delivered earlier" should { "be handled first if the priorities are the same" in { val scheduler = system.actorOf(Props(classOf[PriorityScheduler])) - val request1 = ResourceRequest(Resource(40), WorkerId.unspecified, Priority.HIGH, Relaxation.ANY) - val request2 = ResourceRequest(Resource(20), WorkerId.unspecified, Priority.HIGH, Relaxation.ANY) + val request1 = ResourceRequest(Resource(40), WorkerId.unspecified, HIGH, Relaxation.ANY) + val request2 = ResourceRequest(Resource(20), WorkerId.unspecified, HIGH, Relaxation.ANY) scheduler.tell(RequestResource(appId, request1), mockAppMaster.ref) scheduler.tell(RequestResource(appId, request2), mockAppMaster.ref) scheduler.tell(WorkerRegistered(workerId1, MasterInfo.empty), mockWorker1.ref) scheduler.tell(ResourceUpdate(mockWorker1.ref, workerId1, Resource(100)), mockWorker1.ref) - mockAppMaster.expectMsg(5 seconds, ResourceAllocated(Array(ResourceAllocation(Resource(40), mockWorker1.ref, workerId1)))) - mockAppMaster.expectMsg(5 seconds, ResourceAllocated(Array(ResourceAllocation(Resource(20), mockWorker1.ref, workerId1)))) + var expect = ResourceAllocated( + Array(ResourceAllocation(Resource(40), mockWorker1.ref, workerId1))) + mockAppMaster.expectMsgPF(5.seconds) { + case request: ResourceAllocated if sameElement(request, expect) => Unit + } + expect = ResourceAllocated( + Array(ResourceAllocation(Resource(20), mockWorker1.ref, workerId1))) + mockAppMaster.expectMsgPF(5.seconds) { + case request: ResourceAllocated if sameElement(request, expect) => Unit + } } } "The PriorityScheduler" should { "handle the resource request with different relaxation" in { val scheduler = system.actorOf(Props(classOf[PriorityScheduler])) - val request1 = ResourceRequest(Resource(40), workerId2, Priority.HIGH, Relaxation.SPECIFICWORKER) - val request2 = ResourceRequest(Resource(20), workerId1, Priority.NORMAL, Relaxation.SPECIFICWORKER) + val request1 = ResourceRequest(Resource(40), workerId2, HIGH, Relaxation.SPECIFICWORKER) + val request2 = ResourceRequest(Resource(20), workerId1, NORMAL, Relaxation.SPECIFICWORKER) scheduler.tell(RequestResource(appId, request1), mockAppMaster.ref) scheduler.tell(RequestResource(appId, request2), mockAppMaster.ref) scheduler.tell(WorkerRegistered(workerId1, MasterInfo.empty), mockWorker1.ref) scheduler.tell(ResourceUpdate(mockWorker1.ref, workerId1, Resource(100)), mockWorker1.ref) - mockAppMaster.expectMsg(5 seconds, ResourceAllocated(Array(ResourceAllocation(Resource(20), mockWorker1.ref, workerId1)))) + + var expect = ResourceAllocated( + Array(ResourceAllocation(Resource(20), mockWorker1.ref, workerId1))) + mockAppMaster.expectMsgPF(5.seconds) { + case request: ResourceAllocated if sameElement(request, expect) => Unit + } scheduler.tell(WorkerRegistered(workerId2, MasterInfo.empty), mockWorker2.ref) scheduler.tell(ResourceUpdate(mockWorker2.ref, workerId2, Resource(100)), mockWorker2.ref) - mockAppMaster.expectMsg(5 seconds, ResourceAllocated(Array(ResourceAllocation(Resource(40), mockWorker2.ref, workerId2)))) - val request3 = ResourceRequest(Resource(30), WorkerId.unspecified, Priority.NORMAL, Relaxation.ANY, executorNum = 2) + expect = ResourceAllocated( + Array(ResourceAllocation(Resource(40), mockWorker2.ref, workerId2))) + mockAppMaster.expectMsgPF(5.seconds) { + case request: ResourceAllocated if sameElement(request, expect) => Unit + } + + val request3 = ResourceRequest( + Resource(30), WorkerId.unspecified, NORMAL, Relaxation.ANY, executorNum = 2) scheduler.tell(RequestResource(appId, request3), mockAppMaster.ref) - mockAppMaster.expectMsg(5 seconds, ResourceAllocated(Array(ResourceAllocation(Resource(15), mockWorker1.ref, workerId1), ResourceAllocation(Resource(15), mockWorker2.ref, workerId2)))) - //we have to manually update the resource on each worker + expect = ResourceAllocated(Array( + ResourceAllocation(Resource(15), mockWorker1.ref, workerId1), + ResourceAllocation(Resource(15), mockWorker2.ref, workerId2))) + mockAppMaster.expectMsgPF(5.seconds) { + case request: ResourceAllocated if sameElement(request, expect) => Unit + } + + // We have to manually update the resource on each worker scheduler.tell(ResourceUpdate(mockWorker1.ref, workerId1, Resource(65)), mockWorker1.ref) scheduler.tell(ResourceUpdate(mockWorker2.ref, workerId2, Resource(45)), mockWorker2.ref) - val request4 = ResourceRequest(Resource(60), WorkerId(0, 0L), Priority.NORMAL, Relaxation.ONEWORKER) + val request4 = ResourceRequest(Resource(60), WorkerId(0, 0L), NORMAL, Relaxation.ONEWORKER) scheduler.tell(RequestResource(appId, request4), mockAppMaster.ref) - mockAppMaster.expectMsg(5 seconds, ResourceAllocated(Array(ResourceAllocation(Resource(60), mockWorker1.ref, workerId1)))) + + expect = ResourceAllocated( + Array(ResourceAllocation(Resource(60), mockWorker1.ref, workerId1))) + mockAppMaster.expectMsgPF(5.seconds) { + case request: ResourceAllocated if sameElement(request, expect) => Unit + } } } @@ -144,7 +199,7 @@ class PrioritySchedulerSpec(_system: ActorSystem) extends TestKit(_system) with scheduler.tell(WorkerRegistered(workerId2, MasterInfo.empty), mockWorker2.ref) scheduler.tell(ResourceUpdate(mockWorker2.ref, workerId2, Resource(100)), mockWorker2.ref) - //By default, the request requires only one executor + // By default, the request requires only one executor val request2 = ResourceRequest(Resource(20), WorkerId.unspecified) scheduler.tell(RequestResource(appId, request2), mockAppMaster.ref) val allocations2 = mockAppMaster.receiveN(1).head.asInstanceOf[ResourceAllocated] @@ -157,7 +212,7 @@ class PrioritySchedulerSpec(_system: ActorSystem) extends TestKit(_system) with assert(allocations3.allocations.length == 3) assert(allocations3.allocations.forall(_.resource == Resource(8))) - //The total available resource can not satisfy the requirements with executor number + // The total available resource can not satisfy the requirements with executor number scheduler.tell(ResourceUpdate(mockWorker1.ref, workerId1, Resource(30)), mockWorker1.ref) scheduler.tell(ResourceUpdate(mockWorker2.ref, workerId2, Resource(30)), mockWorker2.ref) val request4 = ResourceRequest(Resource(60), WorkerId.unspecified, executorNum = 3) @@ -166,7 +221,7 @@ class PrioritySchedulerSpec(_system: ActorSystem) extends TestKit(_system) with assert(allocations4.allocations.length == 2) assert(allocations4.allocations.forall(_.resource == Resource(20))) - //When new resources are available, the remaining request will be satisfied + // When new resources are available, the remaining request will be satisfied scheduler.tell(ResourceUpdate(mockWorker1.ref, workerId1, Resource(40)), mockWorker1.ref) val allocations5 = mockAppMaster.receiveN(1).head.asInstanceOf[ResourceAllocated] assert(allocations5.allocations.length == 1) diff --git a/daemon/src/test/scala/io/gearpump/cluster/worker/WorkerSpec.scala b/daemon/src/test/scala/io/gearpump/cluster/worker/WorkerSpec.scala index 9fc409621..46e8d3799 100644 --- a/daemon/src/test/scala/io/gearpump/cluster/worker/WorkerSpec.scala +++ b/daemon/src/test/scala/io/gearpump/cluster/worker/WorkerSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,13 +17,14 @@ */ package io.gearpump.cluster.worker +import scala.concurrent.Await +import scala.concurrent.duration._ + import akka.actor.{ActorSystem, PoisonPill, Props} import akka.testkit.TestProbe -import com.typesafe.config.ConfigFactory -import io.gearpump.WorkerId -import io.gearpump.cluster.WorkerToMaster.RegisterNewWorker -import io.gearpump.cluster.{TestUtil, MasterHarness} -import io.gearpump.util.{ActorSystemBooter, ActorUtil, Constants} +import com.typesafe.config.{Config, ConfigFactory} +import org.scalatest._ + import io.gearpump.cluster.AppMasterToWorker.{ChangeExecutorResource, LaunchExecutor, ShutdownExecutor} import io.gearpump.cluster.MasterToWorker.{UpdateResourceFailed, WorkerRegistered} import io.gearpump.cluster.WorkerToAppMaster.{ExecutorLaunchRejected, ShutdownExecutorFailed, ShutdownExecutorSucceed} @@ -32,13 +33,9 @@ import io.gearpump.cluster.master.Master.MasterInfo import io.gearpump.cluster.scheduler.Resource import io.gearpump.cluster.{ExecutorJVMConfig, MasterHarness, TestUtil} import io.gearpump.util.{ActorSystemBooter, ActorUtil, Constants} -import org.scalatest._ - -import scala.concurrent.duration._ -import scala.language.postfixOps class WorkerSpec extends WordSpec with Matchers with BeforeAndAfterEach with MasterHarness { - override def config = TestUtil.DEFAULT_CONFIG + override def config: Config = TestUtil.DEFAULT_CONFIG val appId = 1 val workerId: WorkerId = WorkerId(1, 0L) @@ -48,14 +45,14 @@ class WorkerSpec extends WordSpec with Matchers with BeforeAndAfterEach with Mas var client: TestProbe = null val workerSlots = 50 - override def beforeEach() = { + override def beforeEach(): Unit = { startActorSystem() mockMaster = TestProbe()(getActorSystem) masterProxy = TestProbe()(getActorSystem) client = TestProbe()(getActorSystem) } - override def afterEach() = { + override def afterEach(): Unit = { shutdownActorSystem() } @@ -64,7 +61,7 @@ class WorkerSpec extends WordSpec with Matchers with BeforeAndAfterEach with Mas val worker = getActorSystem.actorOf(Props(classOf[Worker], mockMaster.ref)) mockMaster watch worker mockMaster.expectMsg(RegisterNewWorker) - mockMaster.expectTerminated(worker, 60 seconds) + mockMaster.expectTerminated(worker, 60.seconds) } } @@ -80,10 +77,11 @@ class WorkerSpec extends WordSpec with Matchers with BeforeAndAfterEach with Mas worker.tell(WorkerRegistered(workerId, MasterInfo(mockMaster.ref)), mockMaster.ref) mockMaster.expectMsg(ResourceUpdate(worker, workerId, Resource(workerSlots))) - worker.tell(UpdateResourceFailed("Test resource update failed", new Exception()), mockMaster.ref) - mockMaster.expectTerminated(worker, 5 seconds) - workerSystem.shutdown() - workerSystem.awaitTermination() + worker.tell( + UpdateResourceFailed("Test resource update failed", new Exception()), mockMaster.ref) + mockMaster.expectTerminated(worker, 5.seconds) + workerSystem.terminate() + Await.result(workerSystem.whenTerminated, Duration.Inf) } } @@ -96,11 +94,17 @@ class WorkerSpec extends WordSpec with Matchers with BeforeAndAfterEach with Mas mockMaster.expectMsg(ResourceUpdate(worker, workerId, Resource(100))) val executorName = ActorUtil.actorNameForExecutor(appId, executorId) - val reportBack = "dummy" //This is an actor path which the ActorSystemBooter will report back to, not needed in this test. - val executionContext = ExecutorJVMConfig(Array.empty[String], getActorSystem.settings.config.getString(Constants.GEARPUMP_APPMASTER_ARGS).split(" "), classOf[ActorSystemBooter].getName, Array(executorName, reportBack), None, username = "user") - - //Test LaunchExecutor - worker.tell(LaunchExecutor(appId, executorId, Resource(101), executionContext), mockMaster.ref) + // This is an actor path which the ActorSystemBooter will report back to, + // not needed in this test + val reportBack = "dummy" + val executionContext = ExecutorJVMConfig(Array.empty[String], + getActorSystem.settings.config.getString(Constants.GEARPUMP_APPMASTER_ARGS).split(" "), + classOf[ActorSystemBooter].getName, Array(executorName, reportBack), None, + username = "user") + + // Test LaunchExecutor + worker.tell(LaunchExecutor(appId, executorId, Resource(101), executionContext), + mockMaster.ref) mockMaster.expectMsg(ExecutorLaunchRejected("There is no free resource on this machine")) worker.tell(LaunchExecutor(appId, executorId, Resource(5), executionContext), mockMaster.ref) @@ -109,13 +113,14 @@ class WorkerSpec extends WordSpec with Matchers with BeforeAndAfterEach with Mas worker.tell(ChangeExecutorResource(appId, executorId, Resource(2)), client.ref) mockMaster.expectMsg(ResourceUpdate(worker, workerId, Resource(98))) - //Test terminationWatch + // Test terminationWatch worker.tell(ShutdownExecutor(appId, executorId, "Test shut down executor"), client.ref) mockMaster.expectMsg(ResourceUpdate(worker, workerId, Resource(100))) client.expectMsg(ShutdownExecutorSucceed(1, 1)) worker.tell(ShutdownExecutor(appId, executorId + 1, "Test shut down executor"), client.ref) - client.expectMsg(ShutdownExecutorFailed(s"Can not find executor ${executorId + 1} for app $appId")) + client.expectMsg(ShutdownExecutorFailed( + s"Can not find executor ${executorId + 1} for app $appId")) mockMaster.ref ! PoisonPill masterProxy.expectMsg(RegisterWorker(workerId)) diff --git a/daemon/src/test/scala/io/gearpump/util/FileServerSpec.scala b/daemon/src/test/scala/io/gearpump/util/FileServerSpec.scala index c2ca6693e..66c7c1db9 100644 --- a/daemon/src/test/scala/io/gearpump/util/FileServerSpec.scala +++ b/daemon/src/test/scala/io/gearpump/util/FileServerSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,30 +20,29 @@ package io.gearpump.util import java.io.File import java.util.concurrent.TimeUnit +import scala.concurrent.Await +import scala.concurrent.duration.Duration + +import akka.actor.ActorSystem +import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} -import akka.actor.{ActorSystem, Props} -import akka.pattern.ask +import io.gearpump.cluster.TestUtil import io.gearpump.google.common.io.Files -import io.gearpump.cluster.{ClusterConfig, TestUtil} import io.gearpump.jarstore.FilePath import io.gearpump.util.FileServer._ -import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} - -import scala.concurrent.duration.Duration -import scala.concurrent.{Await, Future} -import scala.util.Success -class FileServerSpec extends WordSpecLike with Matchers with BeforeAndAfterAll { +class FileServerSpec extends WordSpecLike with Matchers with BeforeAndAfterAll { implicit val timeout = akka.util.Timeout(25, TimeUnit.SECONDS) val host = "localhost" + private val LOG = LogUtil.getLogger(getClass) var system: ActorSystem = null override def afterAll { if (null != system) { - system.shutdown() - system.awaitTermination() + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } } @@ -57,7 +56,7 @@ class FileServerSpec extends WordSpecLike with Matchers with BeforeAndAfterAll FileUtils.writeByteArrayToFile(file, data) val future = client.upload(file) import scala.concurrent.duration._ - val path = Await.result(future, 30 seconds) + val path = Await.result(future, 30.seconds) file.delete() path } @@ -66,7 +65,7 @@ class FileServerSpec extends WordSpecLike with Matchers with BeforeAndAfterAll val file = File.createTempFile("fileserverspec", "test") val future = client.download(remote, file) import scala.concurrent.duration._ - val data = Await.result(future, 10 seconds) + val data = Await.result(future, 10.seconds) val bytes = FileUtils.readFileToByteArray(file) file.delete() @@ -81,7 +80,7 @@ class FileServerSpec extends WordSpecLike with Matchers with BeforeAndAfterAll val server = new FileServer(system, host, 0, rootDir) val port = Await.result((server.start), Duration(25, TimeUnit.SECONDS)) - println("start test web server on port " + port) + LOG.info("start test web server on port " + port) val sizes = List(1, 100, 1000000, 50000000) val client = new Client(system, host, port.port) @@ -113,7 +112,7 @@ class FileServerSpec extends WordSpecLike with Matchers with BeforeAndAfterAll } } - private def randomBytes(size : Int) : Array[Byte] = { + private def randomBytes(size: Int): Array[Byte] = { val bytes = new Array[Byte](size) (new java.util.Random()).nextBytes(bytes) bytes diff --git a/docs/README.md b/docs/README.md index 95fc3801b..e0793eedc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,12 +5,19 @@ have docs corresponding to your checked out version. # Requirements +You need to install ruby and ruby-dev first. On Ubuntu, you ca run command like this: + + sudo apt-get install ruby + sudo apt-get install ruby-dev + sudo apt-get install python-setuptools + We use Markdown to write and Jekyll to translate the documentation to static HTML. You can install all needed software via: - gem install jekyll - gem install kramdown - gem install html-proofer + sudo gem install jekyll + sudo gem install kramdown + sudo gem install html-proofer + sudo gem install pygments.rb sudo easy_install Pygments For Mac OSX you may need to do `sudo gem install -n /usr/local/bin jekyll` if you see the following error: @@ -22,17 +29,10 @@ ERROR: While executing gem ... (Errno::EPERM) Kramdown is needed for Markdown processing and the Python based Pygments is used for syntax highlighting. -# How to Test - -Command `jekyll build` can be used to make a test build. - -Command `jekyll serve --watch` can be used for debug purpose. Jekyll will start a web server at -`localhost:4000` and watch the docs directory for updates. Use this mode to experiment commits and check changes locally. - # How to Build Command `./build_doc.sh` can be used to create a full document folder under site/. -# Contribute +# How to contribute The documentation pages are written in [Markdown](http://daringfireball.net/projects/markdown/syntax). It is possible to use the @@ -52,7 +52,7 @@ Furthermore, you can access variables found in `docs/_config.yml` as follows: This will be replaced with the value of the variable called `NAME` when generating the docs. -All documents are structed with headings. From these heading, a page outline is +All documents are structured with headings. From these heading, a page outline is automatically generated for each page. ``` @@ -66,3 +66,10 @@ automatically generated for each page. Please stick to the "logical order" when using the headlines, e.g. start with level-2 headings and use level-3 headings for subsections, etc. Don't use a different ordering, because you don't like how a headline looks. + +# How to Test + +Command `jekyll build` can be used to make a test build. + +Command `jekyll serve --watch` can be used for debug purpose. Jekyll will start a web server at +`localhost:4000` and watch the docs directory for updates. Use this mode to experiment commits and check changes locally. diff --git a/docs/_layouts/404.html b/docs/_layouts/404.html index 044654413..1bfbe2185 100644 --- a/docs/_layouts/404.html +++ b/docs/_layouts/404.html @@ -1,157 +1,159 @@ - - - Page Not Found :( - - - -

-

Not found :(

-

Sorry, but the page you were trying to view does not exist.

-

It looks like this was the result of either:

-
    -
  • a mistyped address
    • -
  • an out-of-date link
    • -
- - -
- + + + Page Not Found :( + + + +
+

Not found :(

+ +

Sorry, but the page you were trying to view does not exist.

+ +

It looks like this was the result of either:

+
    +
  • a mistyped address
    • +
  • an out-of-date link
    • +
+ + +
+ diff --git a/docs/_layouts/global.html b/docs/_layouts/global.html index d17a55a77..d5c7a2078 100644 --- a/docs/_layouts/global.html +++ b/docs/_layouts/global.html @@ -1,20 +1,25 @@ - - - - + + + + + {{ page.title }} - Gearpump {{site.GEARPUMP_VERSION}} Documentation {% if page.description %} - + {% endif %} {% if page.redirect %} - - + + {% endif %} @@ -29,139 +34,149 @@ - + - -
- {% if page.displayTitle %} -

{{ page.displayTitle }}

- {% else %} -

{{ page.title }}

- {% endif %} +
+ {% if page.displayTitle %} +

{{ page.displayTitle }}

+ {% else %} +

{{ page.title }}

+ {% endif %} - {{ content }} + {{ content }} -
+
+ - - - - + + + + - - - + + + diff --git a/docs/build_doc.sh b/docs/build_doc.sh index c8f691eed..72a2079b9 100755 --- a/docs/build_doc.sh +++ b/docs/build_doc.sh @@ -32,7 +32,7 @@ export BUILD_API=$2 # generate _site documents jekyll build -# check html link validality +# check html link validity echo "Checking generated HTMLs..." htmlproof _site \ --disable-external \ @@ -46,7 +46,6 @@ if [ "$BUILD_API" = 1 ]; then echo "Running 'sbt clean unidoc'; this may take a few minutes..." cd $CURDIR/.. sbt clean unidoc - echo "Moving back into docs dir." cd $CURDIR diff --git a/docs/css/api-javadocs.css b/docs/css/api-javadocs.css index 832e92609..e48ebcfe1 100644 --- a/docs/css/api-javadocs.css +++ b/docs/css/api-javadocs.css @@ -35,8 +35,8 @@ padding-right: 9px; padding-left: 9px; -webkit-border-radius: 9px; - -moz-border-radius: 9px; - border-radius: 9px; + -moz-border-radius: 9px; + border-radius: 9px; } .developer { diff --git a/docs/css/main.css b/docs/css/main.css index f38070124..781d05b35 100644 --- a/docs/css/main.css +++ b/docs/css/main.css @@ -42,7 +42,7 @@ h3 { * using solution at http://stackoverflow.com/questions/8878033/how- * to-make-twitter-bootstrap-menu-dropdown-on-hover-rather-than-click **/ -ul.nav li.dropdown:hover ul.dropdown-menu{ +ul.nav li.dropdown:hover ul.dropdown-menu { display: block; } @@ -84,26 +84,39 @@ ul.nav li.dropdown ul.dropdown-menu li.dropdown-submenu ul.dropdown-menu { /** * MathJax (embedded latex formulas) */ -.MathJax .mo { color: inherit } -.MathJax .mi { color: inherit } -.MathJax .mf { color: inherit } -.MathJax .mh { color: inherit } +.MathJax .mo { + color: inherit +} + +.MathJax .mi { + color: inherit +} + +.MathJax .mf { + color: inherit +} + +.MathJax .mh { + color: inherit +} /** * AnchorJS (anchor links when hovering over headers) */ -a.anchorjs-link:hover { text-decoration: none; } +a.anchorjs-link:hover { + text-decoration: none; +} /** * Dashboard Look And Feel Adjustments */ * { - font-family: roboto,"Helvetica Neue",Helvetica,Arial,sans-serif; + font-family: roboto, "Helvetica Neue", Helvetica, Arial, sans-serif; } pre, pre > *, code, code > * { - font-family: "roboto mono",monaco,consolas,menlo,"Lucida Console",monospace !important; + font-family: "roboto mono", monaco, consolas, menlo, "Lucida Console", monospace !important; font-size: 12px; font-weight: normal !important; line-height: 165%; diff --git a/docs/deployment-configuration.md b/docs/deployment-configuration.md index d0ef37d4f..70b550010 100644 --- a/docs/deployment-configuration.md +++ b/docs/deployment-configuration.md @@ -29,7 +29,7 @@ gearpump{ ## Configuration for user submitted application job -For user application job, it will read configuration file `gear.conf` and `application.conf` from classpath, while `application.conf` has higher prioprty. +For user application job, it will read configuration file `gear.conf` and `application.conf` from classpath, while `application.conf` has higher priority. The default classpath contains: 1. `conf/` @@ -37,7 +37,6 @@ The default classpath contains: For example, you can put a `application.conf` on your working directory, and then it will be effective when you submit a new job application. - ## Logging To change the log level, you need to change both `gear.conf`, and `log4j.properties`. diff --git a/docs/deployment-msg-delivery.md b/docs/deployment-msg-delivery.md index ff6de1fa0..10208ba5b 100644 --- a/docs/deployment-msg-delivery.md +++ b/docs/deployment-msg-delivery.md @@ -3,10 +3,9 @@ layout: global title: Deployment for Reliable Message Delivery --- - ## How to deploy for At Least Once Message Delivery? -As introduced in the [What is At Least Once Message Delievery](message-delivery.html#what-is-at-least-once-message-delivery), Gearpump has a built in KafkaSource. To get at least once message delivery, users should deploy a Kafka cluster as the offset store along with the Gearpump cluster. +As introduced in the [What is At Least Once Message Delivery](message-delivery.html#what-is-at-least-once-message-delivery), Gearpump has a built in KafkaSource. To get at least once message delivery, users should deploy a Kafka cluster as the offset store along with the Gearpump cluster. Here's an example to deploy a local Kafka cluster. @@ -31,7 +30,6 @@ Here's an example to deploy a local Kafka cluster. val source = new KafkaSource("topic1", "localhost:2181", offsetStorageFactory) ``` - ## How to deploy for Exactly Once Message Delivery? Exactly Once Message Delivery requires both an offset store and a checkpoint store. For the offset store, a Kafka cluster should be deployed as in the previous section. As for the checkpoint store, Gearpump has built-in support for Hadoop file systems, like HDFS. Hence, users should deploy a HDFS cluster alongside the Gearpump cluster. diff --git a/docs/deployment-resource-isolation.md b/docs/deployment-resource-isolation.md index 780dd609b..0cb37641a 100644 --- a/docs/deployment-resource-isolation.md +++ b/docs/deployment-resource-isolation.md @@ -9,7 +9,7 @@ cgroups (abbreviated from control groups) is a Linux kernel feature to limit, ac ## Start CGroup Service -Cgroups feature is only supported by Linux whose kernel version is larger than 2.6.18. Please also make sure the SELinux is disabled before start CGroup. +CGroup feature is only supported by Linux whose kernel version is larger than 2.6.18. Please also make sure the SELinux is disabled before start CGroup. The following steps are supposed to be executed by root user. @@ -30,7 +30,7 @@ The following steps are supposed to be executed by root user. perf_event /sys/fs/cgroup/perf_event ``` -3. If you want to assign permission to user **gear** to launch Gearpump Worker and applications with resouce isolation enabled, you need to check gear's uid and gid in /etc/passwd file, let's take **500** for example. +3. If you want to assign permission to user **gear** to launch Gearpump Worker and applications with resource isolation enabled, you need to check gear's uid and gid in /etc/passwd file, let's take **500** for example. 4. Add following content to /etc/cgconfig.conf @@ -71,7 +71,7 @@ The following steps are supposed to be executed by root user. 5. There should be a folder **gearpump** generated under the mount point of cpu subsystem and its owner is **gear:gear**. -6. Repeat the above-mentioned steps on each machine where you want to lauch Gearpump. +6. Repeat the above-mentioned steps on each machine where you want to launch Gearpump. ## Enable Cgroups in Gearpump 1. Login into the machine which has CGroup prepared with user **gear**. @@ -90,11 +90,11 @@ The following steps are supposed to be executed by root user. Please note the gearpump.cgroup.root **gearpump** must be consistent with the group name in /etc/cgconfig.conf. -3. Repeat the above-mentioned steps on each machine where you want to lauch Gearpump +3. Repeat the above-mentioned steps on each machine where you want to launch Gearpump 4. Start the Gearpump cluster, please refer to [Deploy Gearpump in Standalone Mode](deployment-standalone.html) -## Launch Application From Commad Line +## Launch Application From Command Line 1. Login into the machine which has Gearpump distribution. 2. Enter into Gearpump's home folder, edit gear.conf under folder ```${GEARPUMP_HOME}/conf/``` @@ -117,4 +117,4 @@ The following steps are supposed to be executed by root user. If you want to submit the application from dashboard, by default the ```gearpump.cgroup.cpu-core-limit-per-executor``` is inherited from Worker's configuration. You can provide your own conf file to override it. ## Limitations -Windows and Mac OS X don't support CGroup, so the resource isolation will not work even if you turn it on. There will not be any limitition for single executor's cpu usage. \ No newline at end of file +Windows and Mac OS X don't support CGroup, so the resource isolation will not work even if you turn it on. There will not be any limitation for single executor's cpu usage. \ No newline at end of file diff --git a/docs/deployment-security.md b/docs/deployment-security.md index 4be134654..9ce3715a5 100644 --- a/docs/deployment-security.md +++ b/docs/deployment-security.md @@ -36,7 +36,7 @@ Suppose user `gear` will launch gearpump on YARN, then the corresponding princip Before calling "yarnclient launch", make sure you have put all yarn configuration files under classpath. Typically, you can just copy all files under `$HADOOP_HOME/etc/hadoop` from one of the YARN cluster machine to `conf/yarnconf` of gearpump. `$HADOOP_HOME` points to the Hadoop installation directory. -5. Get Kerberos credentials to sumbit the job: +5. Get Kerberos credentials to submit the job: ``` kinit gearpump/fully.qualified.domain.name@YOUR-REALM.COM @@ -79,7 +79,7 @@ Since Gearpump’s Master-Worker structure is similar to HDFS’s NameNode-DataN 1. User creates kerberos principal and keytab for Gearpump. 2. Deploy the keytab files to all the cluster nodes. -3. Configure Gearpump’s conf file, specify kerberos principal and local keytab file localtion. +3. Configure Gearpump’s conf file, specify kerberos principal and local keytab file location. 4. Start Master and Worker. Every application has a submitter/user. We will separate the application from different users, like different log folders for different applications. diff --git a/docs/deployment-standalone.md b/docs/deployment-standalone.md index 06e701c3c..d77ffc800 100644 --- a/docs/deployment-standalone.md +++ b/docs/deployment-standalone.md @@ -9,7 +9,6 @@ Standalone mode is a distributed cluster mode. That is, Gearpump runs as service To deploy Gearpump in cluster mode, please first check that the [Pre-requisites](hardware-requirement.html) are met. - ### How to Install You need to have Gearpump binary at hand. Please refer to [How to get gearpump distribution](get-gearpump-distribution.html) to get the Gearpump binary. diff --git a/docs/deployment-ui-authentication.md b/docs/deployment-ui-authentication.md index 04a9c7b0b..872a85725 100644 --- a/docs/deployment-ui-authentication.md +++ b/docs/deployment-ui-authentication.md @@ -5,7 +5,6 @@ title: UI Dashboard Authentication and Authorization ## What is this about? - ## How to enable UI authentication? 1. Change config file gear.conf, find entry `gearpump-ui.gearpump.ui-security.authentication-enabled`, change the value to true @@ -48,7 +47,6 @@ is the steps to configure ConfigFileBasedAuthenticator. For the default authentication plugin, it has three categories of users: admins, users, and guests. - * admins: have unlimited permission, like shutdown a cluster, add/remove machines. * users: have limited permission to submit an application and etc.. * guests: can not submit/kill applications, but can view the application status. @@ -245,7 +243,7 @@ To use Google OAuth2 Authenticator, there are several steps: 3. Config gear.conf `gearpump.ui-security.oauth2-authenticators.cloudfoundryuaa` section. Please make sure class name, client ID, client Secret, and callback URL are set properly. -**NOTE:** The callback URL here should matche what you set on CloudFoundry UAA in step1. +**NOTE:** The callback URL here should match what you set on CloudFoundry UAA in step1. #### Step3: Configure network proxy for Gearpump UI server if applies @@ -286,7 +284,7 @@ You can follow the Google OAuth2 example code to define a custom OAuth2Authentic * parameters and form fields. * * @note '''Thread-safety''' is a MUST requirement. Developer need to ensure the sub-class is thread-safe. - * Sub-class should have a parameterless constructor. + * Sub-class should have a parameter-less constructor. * * @note OAuth2 Authenticator requires access of Internet. Please make sure HTTP proxy are * set properly if applied. @@ -374,4 +372,3 @@ You can follow the Google OAuth2 example code to define a custom OAuth2Authentic ``` The configuration entry is supposed to be used by class `SocialNetworkXAuthenticator`. - diff --git a/docs/dev-connectors.md b/docs/dev-connectors.md index a9983fcbc..08ff023ef 100644 --- a/docs/dev-connectors.md +++ b/docs/dev-connectors.md @@ -9,7 +9,7 @@ title: Gearpump Connectors ### DataSource `DataSource` is the concept in Gearpump that without input and will output messages. So, basically, `DataSource` is the start point of a streaming processing flow. -As Gearpump depends on `DataSource` to be replayable to ensure at-least-once message delivery and exactly-once message delivery, for some data sources, we will need a `io.gearpump.streaming.transaction.api.OffsetStorageFactory` to store the offset (progress) of current `DataSource`. So that, when a replay is needed, Gearpump can guide `DataSource` to replay from certain offset. +As Gearpump depends on `DataSource` to be replay-able to ensure at-least-once message delivery and exactly-once message delivery, for some data sources, we will need a `io.gearpump.streaming.transaction.api.OffsetStorageFactory` to store the offset (progress) of current `DataSource`. So that, when a replay is needed, Gearpump can guide `DataSource` to replay from certain offset. Currently Gearpump `DataSource` only support infinite stream. Finite stream support will be added in a near future release. @@ -34,7 +34,6 @@ Name | Description `HBaseSink` | Write the message to HBase. The message to write must be HBase `Put` or a tuple of `(rowKey, family, column, value)`. `KafkaSink` | Write to Kafka. - ## Use of Connectors ### Use of `KafkaSource` @@ -88,7 +87,6 @@ Then, you can use `KafkaSource` in your application: To use `HBaseSink` in your application, you first need to add the `gearpump-external-hbase` library dependency in your application: - ``` "com.github.intel-hadoop" %% "gearpump-external-hbase" % {{ site.GEARPUMP_VERSION }} ``` @@ -101,7 +99,6 @@ To use `HBaseSink` in your application, you first need to add the `gearpump-exte ``` - To connect to HBase, you need to provide following info: - the HBase configuration to tell which HBase service to connect - the table name (you must create the table yourself, see the [HBase documentation](https://hbase.apache.org/book.html)) diff --git a/docs/dev-custom-serializer.md b/docs/dev-custom-serializer.md index ca0a332d9..5be05440a 100644 --- a/docs/dev-custom-serializer.md +++ b/docs/dev-custom-serializer.md @@ -41,7 +41,6 @@ but rather import io.gearpump.google.common.io.Files ``` - ##### System Level Serializer If the serializer is widely used, you can define a global serializer which is available to all applications(or worker or master) in the system. @@ -128,7 +127,6 @@ gearpump { ###### Step3: All set! - #### Advanced: Choose another serialization framework Note: This is only for advanced user which require deep customization of Gearpump platform. diff --git a/docs/dev-non-streaming-example.md b/docs/dev-non-streaming-example.md index eac92d0c7..1aec2d1ce 100644 --- a/docs/dev-non-streaming-example.md +++ b/docs/dev-non-streaming-example.md @@ -11,7 +11,6 @@ What Distributed Shell do is that user send a shell command to the cluster and t Repository and library dependencies can be found at [Maven Setting](maven-setting.html) - ### Define Executor Class ```scala diff --git a/docs/dev-rest-api.md b/docs/dev-rest-api.md index 7b2bb6fc8..6243a89f2 100644 --- a/docs/dev-rest-api.md +++ b/docs/dev-rest-api.md @@ -579,8 +579,6 @@ Sample Response: {"enabled":true} ``` - - ### GET api/v1.0/supervisor Get the supervisor path @@ -616,7 +614,6 @@ Sample Response: ### POST api/v1.0/supervisor/removeworker/<worker-id> Remove single worker instance by specifying a worker Id. - **NOTE:* Use with caution! **NOTE:** All executors JVMs under this worker JVM will also be destroyed. It will trigger failover for all @@ -637,7 +634,6 @@ Sample Response: ## Application service - ### GET api/v1.0/appmaster/<appId>?detail=<true|false> Query information of an specific application of Id appId @@ -793,7 +789,6 @@ Sample Response: } ``` - ### GET api/v1.0/appmaster/<appId>/config Query the configuration of specific application appId diff --git a/docs/dev-storm.md b/docs/dev-storm.md index 914aabe45..129f38e34 100644 --- a/docs/dev-storm.md +++ b/docs/dev-storm.md @@ -4,7 +4,7 @@ title: Storm Compatibility --- Gearpump provides **binary compatibility** for Apache Storm applications. That is to say, users could easily grab an existing Storm jar and run it -on Gearpump. This documentation illustrates Gearpump's comapatibility with Storm. +on Gearpump. This documentation illustrates Gearpump's compatibility with Storm. ## What Storm features are supported on Gearpump @@ -32,7 +32,7 @@ on Gearpump. This documentation illustrates Gearpump's comapatibility with Storm | storm-jdbc | yes | | storm-redis | yes | | flux | yes | -| storm-eventhubs | not verfied | +| storm-eventhubs | not verified | | Trident | no | ### At Least Once support @@ -95,7 +95,6 @@ This section shows how to run an existing Storm jar in a local Gearpump cluster. a. submit Storm applications through command line - ``` bin/storm app -verbose -config app.yaml -jar storm-starter-${STORM_VERSION}.jar storm.starter.ExclamationTopology exclamation ``` @@ -138,14 +137,14 @@ Here's an example of `WordCountTopology` with acker bolts (ackers) being transla Gearpump creates a `StormProducer` for each Storm spout and a `StormProcessor` for each Storm bolt (except for ackers) with the same parallelism, and wires them together using the same grouping strategy (partitioning in Gearpump) as in Storm. -At runtime, spouts and bolts are running inside `StormProducer` tasks and `StormProcessor` tasks respectively. Messages emitted by spout are passed to `StormProducer`, transferred to `StormProcessor` and passed down to bolt. Messages are serialized / deserialized with Storm serializers. +At runtime, spouts and bolts are running inside `StormProducer` tasks and `StormProcessor` tasks respectively. Messages emitted by spout are passed to `StormProducer`, transferred to `StormProcessor` and passed down to bolt. Messages are serialized / de-serialized with Storm serializers. Storm ackers are dropped since Gearpump has a different mechanism of message tracking and flow control. ### Task execution Each Storm task is executed by a dedicated thread while all Gearpump tasks of an executor share a thread pool. Generally, we can achieve better performance with a shared thread pool. It's possible, however, some tasks block and take up all the threads. In that case, we can -fall back to the Storm way by setting `gearpump.task-dispatcher` to `"gaerpump.single-thread-dispatcher"` in `gear.conf`. +fall back to the Storm way by setting `gearpump.task-dispatcher` to `"gearpump.single-thread-dispatcher"` in `gear.conf`. ### Message tracking @@ -153,7 +152,6 @@ Storm tracks the lineage of each message with ackers to guarantee at-least-once Gearpump [tracks messages between a sender and receiver in an efficient way](gearpump-internals.html#how-do-we-detect-message-loss). Message loss causes the whole application to replay from the [minimum timestamp of all pending messages in the system](gearpump-internals.html#application-clock-and-global-clock-service). - ### Flow control Storm throttles flow rate at spout, which stops sending messages if the number of unacked messages exceeds `topology.max.spout.pending`. @@ -185,7 +183,6 @@ Since StreamCQL already supports Storm, it's straightforward to run StreamCQL ov 3. Go to the installed stream-cql-binary, and change following settings in `conf/streaming-site.xml` with the output Nimbus configs in Step 2. - ```xml streaming.storm.nimbus.host diff --git a/docs/dev-write-1st-app.md b/docs/dev-write-1st-app.md index 738790bdb..1dd47b643 100644 --- a/docs/dev-write-1st-app.md +++ b/docs/dev-write-1st-app.md @@ -55,8 +55,6 @@ object Split { } ``` - - Like Split, every processor extends a `TaskActor`. The `onStart` method is called once before any message comes in; `onNext` method is called to process every incoming message. Note that Gearpump employs the message-driven model and that's why Split sends itself a message at the end of `onStart` and `onNext` to trigger next message processing. #### Sum Processor @@ -157,13 +155,12 @@ object WordCount extends App with ArgumentsParser { We override `options` value and define an array of command line arguments to parse. We want application users to pass in masters' hosts and ports, the parallelism of split and sum tasks, and how long to run the example. We also specify whether an option is `required` and provide `defaultValue` for some arguments. -Given the `ParseResult` of command line arguments, we create `TaskDescription`s for Split and Sum processors, and connect them with `HashPartitioner` using DAG API. The graph is wrapped in an `AppDescrition` , which is finally submit to master. +Given the `ParseResult` of command line arguments, we create `TaskDescription`s for Split and Sum processors, and connect them with `HashPartitioner` using DAG API. The graph is wrapped in an `AppDescription` , which is finally submit to master. ### Submit application After all these, you need to package everything into a uber jar and submit the jar to Gearpump Cluster. Please check [Application submission tool](commandline.html) to command line tool syntax. - ### Advanced topic For a real application, you definitely need to define your own customized message passing between processors. Customized message needs customized serializer to help message passing over wire. diff --git a/docs/features.md b/docs/features.md index 2bd9417a0..210ac7a50 100644 --- a/docs/features.md +++ b/docs/features.md @@ -7,8 +7,7 @@ description: Gearpump Technical Highlights ### Technical highlights of Gearpump - -Gearpump is a performant, flexible, fault-tolerant, and responsive streaming platform with a lot of nice features, its technical highlights include: +Gearpump is a high performance, flexible, fault-tolerant, and responsive streaming platform with a lot of nice features, its technical highlights include: #### Actors everywhere diff --git a/docs/gearpump-internals.md b/docs/gearpump-internals.md index 922115799..06e045d7b 100644 --- a/docs/gearpump-internals.md +++ b/docs/gearpump-internals.md @@ -4,7 +4,7 @@ displayTitle: Gearpump Internals title: Gearpump Internals description: Gearpump Internals --- -### Actor Hiearachy? +### Actor Hierarchy? ![Actor Hierarchy](img/actor_hierarchy.png) @@ -72,7 +72,7 @@ Without flow control, one task can easily flood another task with too many messa Figure: Flow control, each task is "star" connected to input tasks and output tasks The difficult part for our problem is that each task can have multiple input tasks and output tasks. The input and output must be geared together so that the back pressure can be properly propagated from downstream to upstream. The flow control also needs to consider failures, and it needs to be able to recover when there is message loss. -Another challenge is that the overhead of flow control messages can be big. If we ack every message, there will be huge amount of ack'd messages in the system, degrading streaming performance. The approach we adopted is to use explicit AckRequest message. The target tasks will only ack back when they receive the AckRequest message, and the source will only send AckRequest when it feels necessary. With this approach, we can largely reduce the overhead. +Another challenge is that the overhead of flow control messages can be big. If we ack every message, there will be huge amount of acked messages in the system, degrading streaming performance. The approach we adopted is to use explicit AckRequest message. The target tasks will only ack back when they receive the AckRequest message, and the source will only send AckRequest when it feels necessary. With this approach, we can largely reduce the overhead. ### How do we detect message loss? @@ -231,4 +231,4 @@ When there is message loss, the AppMaster will first pause the global clock serv Kafka queue only expose the offset information for each partition. What KafkaSource do is to maintain its own mapping from Kafka offset to Application timestamp, so that we can map from a application timestamp to a Kafka offset, and replay Kafka messages from that Kafka offset. -The mapping between Application timestmap with Kafka offset is stored in a distributed file system or as a Kafka topic. +The mapping between Application timestamp with Kafka offset is stored in a distributed file system or as a Kafka topic. diff --git a/docs/hardware-requirement.md b/docs/hardware-requirement.md index 87e87c59d..42e839c8b 100644 --- a/docs/hardware-requirement.md +++ b/docs/hardware-requirement.md @@ -25,7 +25,6 @@ CPU | Nothing special HDFS installation | Default is not required. You only need to install it when you want to store the application jars in HDFS. Kafka installation | Default is not required. You need to install Kafka when you want the at-least once message delivery feature. Currently, the only supported data source for this feature is Kafka - ** Table: The default port used in Gearpump:** | usage | Port | Description | diff --git a/docs/how-to-contribute.md b/docs/how-to-contribute.md index 4c15aaf4d..e9e33c19a 100644 --- a/docs/how-to-contribute.md +++ b/docs/how-to-contribute.md @@ -3,7 +3,6 @@ layout: global title: How to contribute --- - ## Contributions Welcome! Gearpump is developed by an open and friendly community. Everybody is cordially welcome to join the community and contribute to Gearpump. There are several ways to interact with the community and to contribute to Gearpump including asking questions, filing bug reports, implementing new use cases, proposing new features, joining discussions on the mailing lists, contributing code or documentation, improving the website, or testing release candidates. @@ -54,7 +53,6 @@ We welcome any contribution to improve our website. Please open an issue at [Gearpump Website Issue Tracker](https://github.com/gearpump/gearpump.github.io/issues) if you think our website could be improved. - ### More ways to contribute… There are many more ways to contribute to the Gearpump community. For example you can diff --git a/docs/index.md b/docs/index.md index 75572c522..d24f3e4d3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -27,12 +27,11 @@ Gearpump's feature set includes: * Samoa compatibility * Both high level and low level API - ### Gearpump Performance Per initial benchmarks we are able to process 18 million messages/second (100 bytes per message) with a 8ms latency on a 4-node cluster. ![Dashboard](img/dashboard.png) ### Gearpump and Akka -Gearump is a 100% Akka based platform. We model big data streaming within the Akka actor hierarchy. +Gearpump is a 100% Akka based platform. We model big data streaming within the Akka actor hierarchy. ![Actor Hierarchy](img/actor_hierarchy.png) diff --git a/docs/js/api-docs.js b/docs/js/api-docs.js index ce89d8943..864502c4e 100644 --- a/docs/js/api-docs.js +++ b/docs/js/api-docs.js @@ -17,7 +17,7 @@ /* Dynamically injected post-processing code for the API docs */ -$(document).ready(function() { +$(document).ready(function () { var annotations = $("dt:contains('Annotations')").next("dd").children("span.name"); addBadges(annotations, "AlphaComponent", ":: AlphaComponent ::", 'Alpha Component'); addBadges(annotations, "DeveloperApi", ":: DeveloperApi ::", 'Developer API'); @@ -29,7 +29,7 @@ function addBadges(allAnnotations, name, tag, html) { var tags = $(".cmt:contains(" + tag + ")") // Remove identifier tags from comments - tags.each(function(index) { + tags.each(function (index) { var oldHTML = $(this).html(); var newHTML = oldHTML.replace(tag, ""); $(this).html(newHTML); diff --git a/docs/js/api-javadocs.js b/docs/js/api-javadocs.js index ead13d6e5..c4eb8a6ec 100644 --- a/docs/js/api-javadocs.js +++ b/docs/js/api-javadocs.js @@ -17,7 +17,7 @@ /* Dynamically injected post-processing code for the API docs */ -$(document).ready(function() { +$(document).ready(function () { addBadges(":: AlphaComponent ::", 'Alpha Component'); addBadges(":: DeveloperApi ::", 'Developer API'); addBadges(":: Experimental ::", 'Experimental'); @@ -27,24 +27,24 @@ function addBadges(tag, html) { var tags = $(".block:contains(" + tag + ")") // Remove identifier tags - tags.each(function(index) { + tags.each(function (index) { var oldHTML = $(this).html(); var newHTML = oldHTML.replace(tag, ""); $(this).html(newHTML); }); // Add html badge tags - tags.each(function(index) { + tags.each(function (index) { if ($(this).parent().is('td.colLast')) { $(this).parent().prepend(html); } else if ($(this).parent('li.blockList') - .parent('ul.blockList') - .parent('div.description') - .parent().is('div.contentContainer')) { + .parent('ul.blockList') + .parent('div.description') + .parent().is('div.contentContainer')) { var contentContainer = $(this).parent('li.blockList') - .parent('ul.blockList') - .parent('div.description') - .parent('div.contentContainer') + .parent('ul.blockList') + .parent('div.description') + .parent('div.contentContainer') var header = contentContainer.prev('div.header'); if (header.length > 0) { header.prepend(html); diff --git a/docs/js/main.js b/docs/js/main.js index 0bac2476b..ae989af07 100644 --- a/docs/js/main.js +++ b/docs/js/main.js @@ -25,7 +25,7 @@ function codeTabs() { "python": "img/python-sm.png", "java": "img/java-sm.png" }; - $("div.codetabs").each(function() { + $("div.codetabs").each(function () { $(this).addClass("tab-content"); // Insert the tab bar @@ -34,7 +34,7 @@ function codeTabs() { // Add each code sample to the tab bar: var codeSamples = $(this).children("div"); - codeSamples.each(function() { + codeSamples.each(function () { $(this).addClass("tab-pane"); var lang = $(this).data("lang"); var image = $(this).data("image"); @@ -44,7 +44,7 @@ function codeTabs() { var id = "tab_" + lang + "_" + counter; $(this).attr("id", id); if (image != null && langImages[lang]) { - var buttonLabel = "" + capitalizedLang + ""; + var buttonLabel = "" + capitalizedLang + ""; } else if (notabs == null) { var buttonLabel = "" + capitalizedLang + ""; } else { @@ -67,12 +67,11 @@ function codeTabs() { $(this).tab('show'); $(document).scrollTop($(this).offset().top - scrollOffset); }); - $("table").each(function() { + $("table").each(function () { $(this).addClass("table table-bordered"); }); } - // A script to fix internal hash links because we have an overlapping top bar. // Based on https://github.com/twitter/bootstrap/issues/193#issuecomment-2281510 function maybeScrollToHash() { @@ -82,7 +81,7 @@ function maybeScrollToHash() { } } -$(function() { +$(function () { codeTabs(); // Display anchor links when hovering over headers. For documentation of the // configuration options, see the AnchorJS documentation. @@ -91,11 +90,15 @@ $(function() { }; anchors.add(); - $(window).bind('hashchange', function() { + $(window).bind('hashchange', function () { maybeScrollToHash(); }); // Scroll now too in case we had opened the page on a hash, but wait a bit because some browsers // will try to do *their* initial scroll after running the onReady handler. - $(window).load(function() { setTimeout(function() { maybeScrollToHash(); }, 25); }); + $(window).load(function () { + setTimeout(function () { + maybeScrollToHash(); + }, 25); + }); }); diff --git a/docs/message-delivery.md b/docs/message-delivery.md index 781bfd5d2..909574f4c 100644 --- a/docs/message-delivery.md +++ b/docs/message-delivery.md @@ -3,16 +3,12 @@ layout: global title: Reliable Message Delivery --- - - ## What is At Least Once Message Delivery? -Messages could be lost on delivery due to network partitions. **At Least Once Message Delivery** (at least once) means the lost messages are delivered one or more times such that at least one is processed and acked by the whole flow. - -Gearpump guarantees at least once for any source that is able to replay message from a past timestamp. In Gearpump, each message is tagged with a timestamp, and the system tracks the minimum timestamp of all pending messages (the global minimum clock). On message loss, application will be restarted to the global minimum clock. Since the source is able to replay from the gloabl minimum clock, all pending messages before the restart will be replayed. Gearpump calls that kind of source `TimeReplayableSource` and already provides a built in -[KafkaSource](gearpump-internals.html#at-least-once-message-delivery-and-kafka). With the KafkaSource to ingest data into Gearpump, users are guaranteed at least once message delievery. - +Messages could be lost on delivery due to network partitions. **At Least Once Message Delivery** (at least once) means the lost messages are delivered one or more times such that at least one is processed and acknowledged by the whole flow. +Gearpump guarantees at least once for any source that is able to replay message from a past timestamp. In Gearpump, each message is tagged with a timestamp, and the system tracks the minimum timestamp of all pending messages (the global minimum clock). On message loss, application will be restarted to the global minimum clock. Since the source is able to replay from the global minimum clock, all pending messages before the restart will be replayed. Gearpump calls that kind of source `TimeReplayableSource` and already provides a built in +[KafkaSource](gearpump-internals.html#at-least-once-message-delivery-and-kafka). With the KafkaSource to ingest data into Gearpump, users are guaranteed at least once message delivery. ## What is Exactly Once Message Delivery? @@ -24,7 +20,6 @@ Users are guaranteed exactly once in Gearpump if they use both a `TimeReplayable ### Persistent API Persistent API consists of `PersistentTask` and `PersistentState`. - Here is an example of using them to keep count of incoming messages. ```scala diff --git a/examples/distributedshell/src/main/scala/io/gearpump/examples/distributedshell/DistShellAppMaster.scala b/examples/distributedshell/src/main/scala/io/gearpump/examples/distributedshell/DistShellAppMaster.scala index 8533a03ce..9b417c0ad 100644 --- a/examples/distributedshell/src/main/scala/io/gearpump/examples/distributedshell/DistShellAppMaster.scala +++ b/examples/distributedshell/src/main/scala/io/gearpump/examples/distributedshell/DistShellAppMaster.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,20 +17,23 @@ */ package io.gearpump.examples.distributedshell +import scala.concurrent.Future + import akka.actor.{Deploy, Props} import akka.pattern.{ask, pipe} import akka.remote.RemoteScope import com.typesafe.config.Config +import org.slf4j.Logger + import io.gearpump.cluster.ClientToMaster.ShutdownApplication import io.gearpump.cluster.appmaster.ExecutorSystemScheduler.{ExecutorSystemJvmConfig, ExecutorSystemStarted, StartExecutorSystemTimeout} import io.gearpump.cluster.{AppDescription, AppMasterContext, ApplicationMaster, ExecutorContext} -import DistShellAppMaster._ +import io.gearpump.examples.distributedshell.DistShellAppMaster._ import io.gearpump.util.{ActorUtil, Constants, LogUtil, Util} -import org.slf4j.Logger -import scala.concurrent.Future +class DistShellAppMaster(appContext: AppMasterContext, app: AppDescription) + extends ApplicationMaster { -class DistShellAppMaster(appContext : AppMasterContext, app : AppDescription) extends ApplicationMaster { import appContext._ import context.dispatcher implicit val timeout = Constants.FUTURE_TIMEOUT @@ -45,10 +48,11 @@ class DistShellAppMaster(appContext : AppMasterContext, app : AppDescription) ex override def receive: Receive = { case ExecutorSystemStarted(executorSystem, _) => import executorSystem.{address, resource => executorResource, worker} - val executorContext = ExecutorContext(currentExecutorId, worker, appId, app.name, self, executorResource) - //start executor + val executorContext = ExecutorContext(currentExecutorId, worker, appId, app.name, + self, executorResource) + // Start executor val executor = context.actorOf(Props(classOf[ShellExecutor], executorContext, app.userConfig) - .withDeploy(Deploy(scope = RemoteScope(address))), currentExecutorId.toString) + .withDeploy(Deploy(scope = RemoteScope(address))), currentExecutorId.toString) executorSystem.bindLifeCycleWith(executor) currentExecutorId += 1 case StartExecutorSystemTimeout => @@ -56,14 +60,17 @@ class DistShellAppMaster(appContext : AppMasterContext, app : AppDescription) ex masterProxy ! ShutdownApplication(appId) context.stop(self) case msg: ShellCommand => - Future.fold(context.children.map(_ ? msg))(new ShellCommandResultAggregator) { (aggregator, response) => - aggregator.aggregate(response.asInstanceOf[ShellCommandResult]) + Future.fold(context.children.map(_ ? msg))(new ShellCommandResultAggregator) { + (aggregator, response) => { + aggregator.aggregate(response.asInstanceOf[ShellCommandResult]) + } }.map(_.toString()) pipeTo sender } private def getExecutorJvmConfig: ExecutorSystemJvmConfig = { val config: Config = app.clusterConfig - val jvmSetting = Util.resolveJvmSetting(config.withFallback(context.system.settings.config)).executor + val jvmSetting = Util.resolveJvmSetting(config.withFallback(context.system.settings.config)) + .executor ExecutorSystemJvmConfig(jvmSetting.classPath, jvmSetting.vmargs, appJar, username, config) } @@ -83,6 +90,6 @@ object DistShellAppMaster { this } - override def toString() = result.toString() + override def toString(): String = result.toString() } } \ No newline at end of file diff --git a/examples/distributedshell/src/main/scala/io/gearpump/examples/distributedshell/DistributedShell.scala b/examples/distributedshell/src/main/scala/io/gearpump/examples/distributedshell/DistributedShell.scala index ac477402e..b7b3eb0e0 100644 --- a/examples/distributedshell/src/main/scala/io/gearpump/examples/distributedshell/DistributedShell.scala +++ b/examples/distributedshell/src/main/scala/io/gearpump/examples/distributedshell/DistributedShell.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,12 +17,14 @@ */ package io.gearpump.examples.distributedshell -import io.gearpump.cluster.{Application, UserConfig, AppDescription} +import org.slf4j.Logger + import io.gearpump.cluster.client.ClientContext -import io.gearpump.cluster.main.{ParseResult, CLIOption, ArgumentsParser} +import io.gearpump.cluster.main.{ArgumentsParser, CLIOption} +import io.gearpump.cluster.{Application, UserConfig} import io.gearpump.util.{AkkaApp, LogUtil} -import org.slf4j.Logger +/** Demo application to distribute and execute shell command on all machines of the cluster */ object DistributedShell extends AkkaApp with ArgumentsParser { private val LOG: Logger = LogUtil.getLogger(getClass) @@ -31,7 +33,8 @@ object DistributedShell extends AkkaApp with ArgumentsParser { override def main(akkaConf: Config, args: Array[String]): Unit = { LOG.info(s"Distributed shell submitting application...") val context = ClientContext(akkaConf) - val appId = context.submit(Application[DistShellAppMaster]("DistributedShell", UserConfig.empty)) + val appId = context.submit(Application[DistShellAppMaster]("DistributedShell", + UserConfig.empty)) context.close() LOG.info(s"Distributed Shell Application started with appId $appId !") } diff --git a/examples/distributedshell/src/main/scala/io/gearpump/examples/distributedshell/DistributedShellClient.scala b/examples/distributedshell/src/main/scala/io/gearpump/examples/distributedshell/DistributedShellClient.scala index feecd1819..cd6f943c8 100644 --- a/examples/distributedshell/src/main/scala/io/gearpump/examples/distributedshell/DistributedShellClient.scala +++ b/examples/distributedshell/src/main/scala/io/gearpump/examples/distributedshell/DistributedShellClient.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,19 +18,19 @@ package io.gearpump.examples.distributedshell import java.util.concurrent.TimeUnit +import scala.concurrent.Await +import scala.concurrent.duration.Duration + +import akka.pattern.ask +import org.slf4j.{Logger, LoggerFactory} import io.gearpump.cluster.client.ClientContext import io.gearpump.cluster.main.{ArgumentsParser, CLIOption} -import DistShellAppMaster.ShellCommand - -import akka.pattern.ask +import io.gearpump.examples.distributedshell.DistShellAppMaster.ShellCommand import io.gearpump.util.{AkkaApp, Constants} -import org.slf4j.{LoggerFactory, Logger} - -import scala.concurrent.Await -import scala.concurrent.duration.Duration -object DistributedShellClient extends AkkaApp with ArgumentsParser { +/** Client to DistributedShell to input "shell command" */ +object DistributedShellClient extends AkkaApp with ArgumentsParser { implicit val timeout = Constants.FUTURE_TIMEOUT private val LOG: Logger = LoggerFactory.getLogger(getClass) diff --git a/examples/distributedshell/src/main/scala/io/gearpump/examples/distributedshell/ShellExecutor.scala b/examples/distributedshell/src/main/scala/io/gearpump/examples/distributedshell/ShellExecutor.scala index 712788c07..2d0fd0652 100644 --- a/examples/distributedshell/src/main/scala/io/gearpump/examples/distributedshell/ShellExecutor.scala +++ b/examples/distributedshell/src/main/scala/io/gearpump/examples/distributedshell/ShellExecutor.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,16 +18,18 @@ package io.gearpump.examples.distributedshell +import scala.sys.process._ +import scala.util.{Failure, Success, Try} + import akka.actor.Actor -import io.gearpump.cluster.{UserConfig, ExecutorContext} -import DistShellAppMaster.{ShellCommandResult, ShellCommand} -import io.gearpump.util.LogUtil import org.slf4j.Logger -import scala.util.{Failure, Success, Try} -import sys.process._ +import io.gearpump.cluster.{ExecutorContext, UserConfig} +import io.gearpump.examples.distributedshell.DistShellAppMaster.{ShellCommand, ShellCommandResult} +import io.gearpump.util.LogUtil -class ShellExecutor(executorContext: ExecutorContext, userConf : UserConfig) extends Actor{ +/** Executor actor on remote machine */ +class ShellExecutor(executorContext: ExecutorContext, userConf: UserConfig) extends Actor { import executorContext._ private val LOG: Logger = LogUtil.getLogger(getClass, executor = executorId, app = appId) @@ -35,7 +37,7 @@ class ShellExecutor(executorContext: ExecutorContext, userConf : UserConfig) ext override def receive: Receive = { case ShellCommand(command) => - val process = Try(s"$command" !!) + val process = Try(s"$command".!!) val result = process match { case Success(msg) => msg case Failure(ex) => ex.getMessage diff --git a/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/DistShellAppMasterSpec.scala b/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/DistShellAppMasterSpec.scala index ec6bba136..2d637341c 100644 --- a/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/DistShellAppMasterSpec.scala +++ b/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/DistShellAppMasterSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,20 +17,24 @@ */ package io.gearpump.examples.distributedshell +import scala.concurrent.Await +import scala.concurrent.duration.Duration + import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestProbe} -import io.gearpump.WorkerId -import io.gearpump.cluster.AppMasterToMaster.{RequestResource, GetAllWorkers, RegisterAppMaster} +import org.scalatest.{BeforeAndAfter, Matchers, WordSpec} + +import io.gearpump.cluster.AppMasterToMaster.{GetAllWorkers, RegisterAppMaster, RequestResource} import io.gearpump.cluster.AppMasterToWorker.LaunchExecutor -import io.gearpump.cluster.MasterToAppMaster.{ResourceAllocated, WorkerList, AppMasterRegistered} +import io.gearpump.cluster.MasterToAppMaster.{AppMasterRegistered, ResourceAllocated, WorkerList} import io.gearpump.cluster._ -import io.gearpump.cluster.appmaster.{AppMasterRuntimeInfo, AppMasterRuntimeEnvironment} -import io.gearpump.cluster.scheduler.{ResourceAllocation, Relaxation, ResourceRequest, Resource} +import io.gearpump.cluster.appmaster.{AppMasterRuntimeEnvironment, AppMasterRuntimeInfo} +import io.gearpump.cluster.scheduler.{Relaxation, Resource, ResourceAllocation, ResourceRequest} +import io.gearpump.cluster.worker.WorkerId import io.gearpump.util.ActorSystemBooter.RegisterActorSystem import io.gearpump.util.ActorUtil -import org.scalatest.{BeforeAndAfter, Matchers, WordSpec} -class DistShellAppMasterSpec extends WordSpec with Matchers with BeforeAndAfter{ +class DistShellAppMasterSpec extends WordSpec with Matchers with BeforeAndAfter { implicit val system = ActorSystem("AppMasterSpec", TestUtil.DEFAULT_CONFIG) val mockMaster = TestProbe()(system) val mockWorker1 = TestProbe()(system) @@ -46,27 +50,29 @@ class DistShellAppMasterSpec extends WordSpec with Matchers with BeforeAndAfter{ "DistributedShell AppMaster" should { "launch one ShellTask on each worker" in { val appMasterInfo = AppMasterRuntimeInfo(appId, appName = appId.toString) - val appMasterContext = AppMasterContext(appId, userName, resource, null, appJar, masterProxy, appMasterInfo) + val appMasterContext = AppMasterContext(appId, userName, resource, null, appJar, + masterProxy, appMasterInfo) TestActorRef[DistShellAppMaster]( AppMasterRuntimeEnvironment.props(List(masterProxy.path), appDescription, appMasterContext)) mockMaster.expectMsgType[RegisterAppMaster] mockMaster.reply(AppMasterRegistered(appId)) - //The DistributedShell AppMaster will ask for worker list + // The DistributedShell AppMaster asks for worker list from Master. mockMaster.expectMsg(GetAllWorkers) mockMaster.reply(WorkerList(workerList)) - //After worker list is ready, DistributedShell AppMaster will request resouce on each worker + // After worker list is ready, DistributedShell AppMaster requests resource on each worker workerList.foreach { workerId => - mockMaster.expectMsg(RequestResource(appId, ResourceRequest(Resource(1), workerId, relaxation = Relaxation.SPECIFICWORKER))) + mockMaster.expectMsg(RequestResource(appId, ResourceRequest(Resource(1), workerId, + relaxation = Relaxation.SPECIFICWORKER))) } - mockMaster.reply(ResourceAllocated(Array(ResourceAllocation(resource, mockWorker1.ref, WorkerId(1, 0L))))) + mockMaster.reply(ResourceAllocated( + Array(ResourceAllocation(resource, mockWorker1.ref, WorkerId(1, 0L))))) mockWorker1.expectMsgClass(classOf[LaunchExecutor]) mockWorker1.reply(RegisterActorSystem(ActorUtil.getSystemAddress(system).toString)) } } after { - system.shutdown() - system.awaitTermination() + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } - } diff --git a/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/DistributedShellClientSpec.scala b/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/DistributedShellClientSpec.scala index adefa4081..973b3b38c 100644 --- a/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/DistributedShellClientSpec.scala +++ b/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/DistributedShellClientSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,19 +17,20 @@ */ package io.gearpump.examples.distributedshell +import scala.concurrent.Future +import scala.util.{Success, Try} + import akka.testkit.TestProbe -import io.gearpump.cluster.ClientToMaster.ResolveAppId -import io.gearpump.cluster.MasterToClient.ResolveAppIdResult -import io.gearpump.cluster.{TestUtil, MasterHarness} -import DistShellAppMaster.ShellCommand -import io.gearpump.util.{LogUtil, Constants, Util} import org.scalatest.{BeforeAndAfter, Matchers, PropSpec} -import scala.concurrent.Future - -import scala.util.{Try, Success} +import io.gearpump.cluster.ClientToMaster.ResolveAppId +import io.gearpump.cluster.MasterToClient.ResolveAppIdResult +import io.gearpump.cluster.{MasterHarness, TestUtil} +import io.gearpump.examples.distributedshell.DistShellAppMaster.ShellCommand +import io.gearpump.util.LogUtil -class DistributedShellClientSpec extends PropSpec with Matchers with BeforeAndAfter with MasterHarness { +class DistributedShellClientSpec + extends PropSpec with Matchers with BeforeAndAfter with MasterHarness { private val LOG = LogUtil.getLogger(getClass) @@ -41,17 +42,19 @@ class DistributedShellClientSpec extends PropSpec with Matchers with BeforeAndAf shutdownActorSystem() } - override def config = TestUtil.DEFAULT_CONFIG + protected override def config = TestUtil.DEFAULT_CONFIG property("DistributedShellClient should succeed to submit application with required arguments") { val command = "ls /" val requiredArgs = Array("-appid", "0", "-command", command) val masterReceiver = createMockMaster() - assert(Try(DistributedShellClient.main(Array.empty[String])).isFailure, "missing required arguments, print usage") - + assert(Try(DistributedShellClient.main(Array.empty[String])).isFailure, + "missing required arguments, print usage") - Future {DistributedShellClient.main(masterConfig, requiredArgs)} + Future { + DistributedShellClient.main(masterConfig, requiredArgs) + } masterReceiver.expectMsg(PROCESS_BOOT_TIME, ResolveAppId(0)) val mockAppMaster = TestProbe()(getActorSystem) diff --git a/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/DistributedShellSpec.scala b/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/DistributedShellSpec.scala index 2a369c403..6eeba580b 100644 --- a/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/DistributedShellSpec.scala +++ b/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/DistributedShellSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,19 +17,19 @@ */ package io.gearpump.examples.distributedshell +import scala.concurrent.Future +import scala.util.Success + import com.typesafe.config.Config -import io.gearpump.cluster.ClientToMaster.SubmitApplication -import io.gearpump.cluster.MasterToClient.SubmitApplicationResult -import io.gearpump.cluster.{TestUtil, MasterHarness} -import io.gearpump.util.Util -import org.scalatest.{BeforeAndAfter, Matchers, PropSpec} import org.scalatest.prop.PropertyChecks +import org.scalatest.{BeforeAndAfter, Matchers, PropSpec} -import scala.util.{Try, Success} - -import scala.concurrent.Future +import io.gearpump.cluster.ClientToMaster.SubmitApplication +import io.gearpump.cluster.MasterToClient.SubmitApplicationResult +import io.gearpump.cluster.{MasterHarness, TestUtil} -class DistributedShellSpec extends PropSpec with PropertyChecks with Matchers with BeforeAndAfter with MasterHarness { +class DistributedShellSpec + extends PropSpec with PropertyChecks with Matchers with BeforeAndAfter with MasterHarness { before { startActorSystem() @@ -46,7 +46,9 @@ class DistributedShellSpec extends PropSpec with PropertyChecks with Matchers wi val masterReceiver = createMockMaster() - Future{DistributedShell.main(masterConfig, requiredArgs)} + Future { + DistributedShell.main(masterConfig, requiredArgs) + } masterReceiver.expectMsgType[SubmitApplication](PROCESS_BOOT_TIME) masterReceiver.reply(SubmitApplicationResult(Success(0))) diff --git a/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/ShellCommandResultAggregatorSpec.scala b/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/ShellCommandResultAggregatorSpec.scala index 6bb11055c..d59981bc4 100644 --- a/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/ShellCommandResultAggregatorSpec.scala +++ b/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/ShellCommandResultAggregatorSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,8 +18,8 @@ package io.gearpump.examples.distributedshell import org.scalatest.{BeforeAndAfter, Matchers, WordSpec} -import DistShellAppMaster.{ShellCommandResultAggregator, ShellCommandResult} +import io.gearpump.examples.distributedshell.DistShellAppMaster.{ShellCommandResult, ShellCommandResultAggregator} class ShellCommandResultAggregatorSpec extends WordSpec with Matchers with BeforeAndAfter { "ShellCommandResultAggregator" should { diff --git a/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/ShellExecutorSpec.scala b/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/ShellExecutorSpec.scala index d51880b74..b301973ed 100644 --- a/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/ShellExecutorSpec.scala +++ b/examples/distributedshell/src/test/scala/io/gearpump/examples/distributedshell/ShellExecutorSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,18 +17,20 @@ */ package io.gearpump.examples.distributedshell +import scala.concurrent.Await +import scala.concurrent.duration.Duration +import scala.sys.process._ +import scala.util.{Failure, Success, Try} + import akka.actor.{ActorSystem, Props} import akka.testkit.TestProbe -import io.gearpump.WorkerId -import io.gearpump.examples.distributedshell.DistShellAppMaster.ShellCommandResult +import org.scalatest.{Matchers, WordSpec} + import io.gearpump.cluster.appmaster.WorkerInfo import io.gearpump.cluster.scheduler.Resource +import io.gearpump.cluster.worker.WorkerId import io.gearpump.cluster.{ExecutorContext, TestUtil, UserConfig} -import DistShellAppMaster.{ShellCommand, ShellCommandResult} -import org.scalatest.{Matchers, WordSpec} - -import scala.sys.process._ -import scala.util.{Failure, Success, Try} +import io.gearpump.examples.distributedshell.DistShellAppMaster.{ShellCommand, ShellCommandResult} class ShellExecutorSpec extends WordSpec with Matchers { @@ -43,10 +45,12 @@ class ShellExecutorSpec extends WordSpec with Matchers { val mockMaster = TestProbe()(system) val worker = TestProbe() val workerInfo = WorkerInfo(workerId, worker.ref) - val executorContext = ExecutorContext(executorId, workerInfo, appId, appName, mockMaster.ref, resource) - val executor = system.actorOf(Props(classOf[ShellExecutor], executorContext, UserConfig.empty)) + val executorContext = ExecutorContext(executorId, workerInfo, appId, appName, + mockMaster.ref, resource) + val executor = system.actorOf(Props(classOf[ShellExecutor], executorContext, + UserConfig.empty)) - val process = Try(s"ls /" !!) + val process = Try(s"ls /".!!) val result = process match { case Success(msg) => msg case Failure(ex) => ex.getMessage @@ -55,8 +59,8 @@ class ShellExecutorSpec extends WordSpec with Matchers { assert(mockMaster.receiveN(1).head.asInstanceOf[ShellCommandResult].equals( ShellCommandResult(executorId, result))) - system.shutdown() - system.awaitTermination() + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } } } diff --git a/examples/distributeservice/README.md b/examples/distributeservice/README.md index da5acac37..82b372664 100644 --- a/examples/distributeservice/README.md +++ b/examples/distributeservice/README.md @@ -15,7 +15,7 @@ In order to run the example: -script ${Script_Path} -serviceName ${Service_Name} -target ${Target_Path} -Dkey1=value1 -Dkey2=value2 ```
This command will distribute the service zip file(variable ```file```) to the target path(variable ```target```), then copy the script to - ```/etc/init.d``` on each machine and install this servcie named with ```serviceName```
+ ```/etc/init.d``` on each machine and install this service named with ```serviceName```
Note that you can pass some variables when the script file is installed, for example, you can submit a script template with syntax like ``` role=${${hostname}.role} diff --git a/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/DistServiceAppMaster.scala b/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/DistServiceAppMaster.scala index 4d3492a7e..a220dc612 100644 --- a/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/DistServiceAppMaster.scala +++ b/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/DistServiceAppMaster.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,21 +18,22 @@ package io.gearpump.experiments.distributeservice import java.io.File +import scala.concurrent.Future import akka.actor.{Deploy, Props} import akka.pattern.{ask, pipe} import akka.remote.RemoteScope import com.typesafe.config.Config +import org.slf4j.Logger + import io.gearpump.cluster.ClientToMaster.ShutdownApplication import io.gearpump.cluster.appmaster.ExecutorSystemScheduler.{ExecutorSystemJvmConfig, ExecutorSystemStarted, StartExecutorSystemTimeout} import io.gearpump.cluster.{AppDescription, AppMasterContext, ApplicationMaster, ExecutorContext} -import DistServiceAppMaster.{FileContainer, GetFileContainer, InstallService} +import io.gearpump.experiments.distributeservice.DistServiceAppMaster.{FileContainer, GetFileContainer, InstallService} import io.gearpump.util._ -import org.slf4j.Logger - -import scala.concurrent.Future -class DistServiceAppMaster(appContext : AppMasterContext, app : AppDescription) extends ApplicationMaster { +class DistServiceAppMaster(appContext: AppMasterContext, app: AppDescription) + extends ApplicationMaster { import appContext._ import context.dispatcher implicit val timeout = Constants.FUTURE_TIMEOUT @@ -42,7 +43,7 @@ class DistServiceAppMaster(appContext : AppMasterContext, app : AppDescription) val rootDirectory = new File("/") val host = context.system.settings.config.getString(Constants.GEARPUMP_HOSTNAME) - val server = context.actorOf(Props(classOf[FileServer], rootDirectory, host , 0)) + val server = context.actorOf(Props(classOf[FileServer], rootDirectory, host, 0)) override def preStart(): Unit = { LOG.info(s"Distribute Service AppMaster started") @@ -54,10 +55,12 @@ class DistServiceAppMaster(appContext : AppMasterContext, app : AppDescription) override def receive: Receive = { case ExecutorSystemStarted(executorSystem, _) => import executorSystem.{address, resource => executorResource, worker} - val executorContext = ExecutorContext(currentExecutorId, worker, appId, app.name, self, executorResource) - //start executor - val executor = context.actorOf(Props(classOf[DistServiceExecutor], executorContext, app.userConfig) - .withDeploy(Deploy(scope = RemoteScope(address))), currentExecutorId.toString) + val executorContext = ExecutorContext(currentExecutorId, worker, + appId, app.name, self, executorResource) + // start executor + val executor = context.actorOf(Props(classOf[DistServiceExecutor], + executorContext, app.userConfig).withDeploy( + Deploy(scope = RemoteScope(address))), currentExecutorId.toString) executorSystem.bindLifeCycleWith(executor) currentExecutorId += 1 case StartExecutorSystemTimeout => @@ -75,7 +78,8 @@ class DistServiceAppMaster(appContext : AppMasterContext, app : AppDescription) private def getExecutorJvmConfig: ExecutorSystemJvmConfig = { val config: Config = app.clusterConfig - val jvmSetting = Util.resolveJvmSetting(config.withFallback(context.system.settings.config)).executor + val jvmSetting = Util.resolveJvmSetting( + config.withFallback(context.system.settings.config)).executor ExecutorSystemJvmConfig(jvmSetting.classPath, jvmSetting.vmargs, appJar, username, config) } @@ -87,10 +91,10 @@ object DistServiceAppMaster { case class FileContainer(url: String) case class InstallService( - url: String, - zipFileName: String, - targetPath: String, - script : Array[Byte], - serviceName: String, - serviceSettings: Map[String, Any]) + url: String, + zipFileName: String, + targetPath: String, + script: Array[Byte], + serviceName: String, + serviceSettings: Map[String, Any]) } diff --git a/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/DistServiceExecutor.scala b/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/DistServiceExecutor.scala index 7fc2a94ad..4a2a8766f 100644 --- a/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/DistServiceExecutor.scala +++ b/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/DistServiceExecutor.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,29 +17,30 @@ */ package io.gearpump.experiments.distributeservice -import java.io.{FileWriter, File} +import java.io.{File, FileWriter} import java.net.InetAddress +import scala.collection.JavaConverters._ +import scala.io.Source +import scala.sys.process._ +import scala.util.{Failure, Success, Try} import akka.actor.Actor import org.apache.commons.io.FileUtils import org.apache.commons.lang.text.StrSubstitutor -import io.gearpump.cluster.{UserConfig, ExecutorContext} -import DistServiceAppMaster.InstallService -import io.gearpump.util.{ActorUtil, LogUtil} import org.slf4j.Logger -import scala.io.Source -import scala.sys.process._ -import collection.JavaConversions._ -import scala.util.{Failure, Success, Try} +import io.gearpump.cluster.{ExecutorContext, UserConfig} +import io.gearpump.experiments.distributeservice.DistServiceAppMaster.InstallService +import io.gearpump.util.{ActorUtil, LogUtil} -class DistServiceExecutor(executorContext: ExecutorContext, userConf : UserConfig) extends Actor { +class DistServiceExecutor(executorContext: ExecutorContext, userConf: UserConfig) extends Actor { import executorContext._ private val LOG: Logger = LogUtil.getLogger(getClass, executor = executorId, app = appId) override def receive: Receive = { case InstallService(url, zipFileName, targetPath, scriptData, serviceName, serviceSettings) => - LOG.info(s"Executor $executorId receive command to install service $serviceName to $targetPath") + LOG.info(s"Executor $executorId receive command to install " + + s"service $serviceName to $targetPath") unzipFile(url, zipFileName, targetPath) installService(scriptData, serviceName, serviceSettings) } @@ -47,31 +48,32 @@ class DistServiceExecutor(executorContext: ExecutorContext, userConf : UserConfi private def unzipFile(url: String, zipFileName: String, targetPath: String) = { val zipFile = File.createTempFile(System.currentTimeMillis().toString, zipFileName) val dir = new File(targetPath) - if(dir.exists()) { + if (dir.exists()) { FileUtils.forceDelete(dir) } val bytes = FileServer.newClient.get(url).get FileUtils.writeByteArrayToFile(zipFile, bytes) - val result = Try(s"unzip ${zipFile.getAbsolutePath} -d $targetPath" !!) + val result = Try(s"unzip ${zipFile.getAbsolutePath} -d $targetPath".!!) result match { case Success(msg) => LOG.info(s"Executor $executorId unzip file to $targetPath") - case Failure(ex) => throw ex + case Failure(ex) => throw ex } } - private def installService(scriptData: Array[Byte], serviceName: String, serviceSettings: Map[String, Any]) = { + private def installService( + scriptData: Array[Byte], serviceName: String, serviceSettings: Map[String, Any]) = { val tempFile = File.createTempFile("gearpump", serviceName) FileUtils.writeByteArrayToFile(tempFile, scriptData) val script = new File("/etc/init.d", serviceName) writeFileWithEnvVariables(tempFile, script, serviceSettings ++ getEnvSettings) - val result = Try(s"chkconfig --add $serviceName" !!) + val result = Try(s"chkconfig --add $serviceName".!!) result match { case Success(msg) => LOG.info(s"Executor install service $serviceName successfully!") case Failure(ex) => throw ex } } - private def getEnvSettings : Map[String, Any] = { + private def getEnvSettings: Map[String, Any] = { Map("workerId" -> worker, "localhost" -> ActorUtil.getSystemAddress(context.system).host.get, "hostname" -> InetAddress.getLocalHost.getHostName) @@ -79,7 +81,7 @@ class DistServiceExecutor(executorContext: ExecutorContext, userConf : UserConfi private def writeFileWithEnvVariables(source: File, target: File, envs: Map[String, Any]) = { val writer = new FileWriter(target) - val sub = new StrSubstitutor(mapAsJavaMap(envs)) + val sub = new StrSubstitutor(envs.asJava) sub.setEnableSubstitutionInVariables(true) Source.fromFile(source).getLines().foreach(line => writer.write(sub.replace(line) + "\r\n")) writer.close() diff --git a/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/DistributeService.scala b/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/DistributeService.scala index d558b5dd6..522dc5e32 100644 --- a/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/DistributeService.scala +++ b/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/DistributeService.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,13 +17,15 @@ */ package io.gearpump.experiments.distributeservice +import org.slf4j.Logger + import io.gearpump.cluster.client.ClientContext -import io.gearpump.cluster.{Application, AppJar, UserConfig, AppDescription} -import io.gearpump.cluster.main.{ParseResult, CLIOption, ArgumentsParser} +import io.gearpump.cluster.main.{ArgumentsParser, CLIOption} +import io.gearpump.cluster.{Application, UserConfig} import io.gearpump.util.{AkkaApp, LogUtil} -import org.slf4j.Logger -object DistributeService extends AkkaApp with ArgumentsParser { +/** Demo app to remotely deploy and start system service on machines in the cluster */ +object DistributeService extends AkkaApp with ArgumentsParser { private val LOG: Logger = LogUtil.getLogger(getClass) override val options: Array[(String, CLIOption[Any])] = Array.empty @@ -31,7 +33,8 @@ object DistributeService extends AkkaApp with ArgumentsParser { override def main(akkaConf: Config, args: Array[String]): Unit = { LOG.info(s"Distribute Service submitting application...") val context = ClientContext(akkaConf) - val appId = context.submit(Application[DistServiceAppMaster]("DistributedService", UserConfig.empty)) + val appId = context.submit(Application[DistServiceAppMaster]("DistributedService", + UserConfig.empty)) context.close() LOG.info(s"Distribute Service Application started with appId $appId !") } diff --git a/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/DistributeServiceClient.scala b/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/DistributeServiceClient.scala index 5f4ebbb85..0d8500142 100644 --- a/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/DistributeServiceClient.scala +++ b/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/DistributeServiceClient.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,31 +18,35 @@ package io.gearpump.experiments.distributeservice import java.io.File -import org.apache.commons.io.FileUtils -import io.gearpump.cluster.client.ClientContext -import io.gearpump.cluster.main.{CLIOption, ArgumentsParser} -import DistServiceAppMaster.{InstallService, FileContainer, GetFileContainer} -import io.gearpump.util.{AkkaApp, LogUtil, FileServer, Constants} -import org.slf4j.{LoggerFactory, Logger} - -import akka.pattern.ask import scala.concurrent.Future import scala.util.{Failure, Success} -object DistributeServiceClient extends AkkaApp with ArgumentsParser{ +import akka.pattern.ask +import org.apache.commons.io.FileUtils + +import io.gearpump.cluster.client.ClientContext +import io.gearpump.cluster.main.{ArgumentsParser, CLIOption} +import io.gearpump.experiments.distributeservice.DistServiceAppMaster.{FileContainer, GetFileContainer, InstallService} +import io.gearpump.util.{AkkaApp, Constants} + +/** Client to submit the service jar */ +object DistributeServiceClient extends AkkaApp with ArgumentsParser { implicit val timeout = Constants.FUTURE_TIMEOUT override val options: Array[(String, CLIOption[Any])] = Array( "appid" -> CLIOption[Int]("", required = true), "file" -> CLIOption[String]("", required = true), - "script" -> CLIOption[String]("", required = true), + "script" -> CLIOption[String]( + "", required = true), "serviceName" -> CLIOption[String]("", required = true), "target" -> CLIOption[String]("", required = true) ) - override def help : Unit = { - super.help + override def help(): Unit = { + super.help() + // scalastyle:off println Console.err.println(s"-D= set a property to the service") + // scalastyle:on println } override def main(akkaConf: Config, args: Array[String]): Unit = { @@ -75,7 +79,7 @@ object DistributeServiceClient extends AkkaApp with ArgumentsParser{ private def parseServiceConfig(args: Array[String]): Map[String, Any] = { val result = Map.empty[String, Any] args.foldLeft(result) { (result, argument) => - if(argument.startsWith("-D") && argument.contains("=")) { + if (argument.startsWith("-D") && argument.contains("=")) { val fixedKV = argument.substring(2).split("=") result + (fixedKV(0) -> fixedKV(1)) } else { diff --git a/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/FileServer.scala b/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/FileServer.scala index 893aa1906..ed0b24d98 100644 --- a/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/FileServer.scala +++ b/examples/distributeservice/src/main/scala/io/gearpump/experiments/distributeservice/FileServer.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,21 +16,20 @@ * limitations under the License. */ - package io.gearpump.experiments.distributeservice import java.io.File +import scala.util.{Failure, Success, Try} import akka.actor.{Actor, Stash} import akka.io.IO -import io.gearpump.util.LogUtil import org.apache.commons.httpclient.HttpClient import org.apache.commons.httpclient.methods.{ByteArrayRequestEntity, GetMethod, PostMethod} import spray.can.Http import spray.http.HttpMethods._ import spray.http._ -import io.gearpump.util.FileUtils -import scala.util.{Failure, Success, Try} + +import io.gearpump.util.{FileUtils, LogUtil} /** * @@ -38,18 +37,18 @@ import scala.util.{Failure, Success, Try} * * port: set port to 0 if you want to bind to random port */ -class FileServer(rootDir: File, host: String, port : Int) extends Actor with Stash { +class FileServer(rootDir: File, host: String, port: Int) extends Actor with Stash { private val LOG = LogUtil.getLogger(getClass) implicit val system = context.system override def preStart(): Unit = { - // create http server + // Creates http server IO(Http) ! Http.Bind(self, host, port) } - override def postStop() : Unit = { - //stop the server + override def postStop(): Unit = { + // Stop the server IO(Http) ! Http.Unbind } @@ -62,14 +61,14 @@ class FileServer(rootDir: File, host: String, port : Int) extends Actor with Sta stash() } - def listen(port : Int) : Receive = { + def listen(port: Int): Receive = { case FileServer.GetPort => { sender ! FileServer.Port(port) } case Http.Connected(remote, _) => sender ! Http.Register(self) - // fetch file from remote uri + // Fetches files from remote uri case HttpRequest(GET, uri, _, _, _) => val child = uri.path.toString() val payload = Try { @@ -83,8 +82,8 @@ class FileServer(rootDir: File, host: String, port : Int) extends Actor with Sta LOG.error("failed to get file " + ex.getMessage) sender ! HttpResponse(status = StatusCodes.InternalServerError, entity = ex.getMessage) } - //save file to remote uri - case post @ HttpRequest(POST, uri, _, _, _) => + // Save file to remote uri + case post@HttpRequest(POST, uri, _, _, _) => val child = uri.path.toString() val status = Try { @@ -104,14 +103,14 @@ class FileServer(rootDir: File, host: String, port : Int) extends Actor with Sta object FileServer { object GetPort - case class Port(port : Int) + case class Port(port: Int) - def newClient = new Client + def newClient: Client = new Client class Client { val client = new HttpClient() - def save(uri : String, data : Array[Byte]) : Try[Int] = { + def save(uri: String, data: Array[Byte]): Try[Int] = { Try { val post = new PostMethod(uri) val entity = new ByteArrayRequestEntity(data) @@ -120,7 +119,7 @@ object FileServer { } } - def get(uri : String) : Try[Array[Byte]] = { + def get(uri: String): Try[Array[Byte]] = { val get = new GetMethod(uri) val status = Try { client.executeMethod(get) diff --git a/examples/distributeservice/src/test/scala/io/gearpump/experiments/distributeservice/DistServiceAppMasterSpec.scala b/examples/distributeservice/src/test/scala/io/gearpump/experiments/distributeservice/DistServiceAppMasterSpec.scala index fcdbf1405..5bafef12c 100644 --- a/examples/distributeservice/src/test/scala/io/gearpump/experiments/distributeservice/DistServiceAppMasterSpec.scala +++ b/examples/distributeservice/src/test/scala/io/gearpump/experiments/distributeservice/DistServiceAppMasterSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,23 +17,25 @@ */ package io.gearpump.experiments.distributeservice +import scala.concurrent.Await +import scala.concurrent.duration._ + import akka.actor.ActorSystem import akka.testkit.{TestActorRef, TestProbe} -import io.gearpump.WorkerId -import io.gearpump.cluster.AppMasterToMaster.{RequestResource, GetAllWorkers, RegisterAppMaster} +import org.scalatest.{BeforeAndAfter, Matchers, WordSpec} + +import io.gearpump.cluster.AppMasterToMaster.{GetAllWorkers, RegisterAppMaster, RequestResource} import io.gearpump.cluster.AppMasterToWorker.LaunchExecutor -import io.gearpump.cluster.MasterToAppMaster.{ResourceAllocated, WorkerList, AppMasterRegistered} -import io.gearpump.cluster.appmaster.{AppMasterRuntimeInfo, AppMasterRuntimeEnvironment} -import io.gearpump.cluster.{AppMasterContext, UserConfig, AppDescription, TestUtil} -import io.gearpump.cluster.scheduler.{ResourceAllocation, Relaxation, ResourceRequest, Resource} -import DistServiceAppMaster.{FileContainer, GetFileContainer} +import io.gearpump.cluster.MasterToAppMaster.{AppMasterRegistered, ResourceAllocated, WorkerList} +import io.gearpump.cluster.appmaster.{AppMasterRuntimeEnvironment, AppMasterRuntimeInfo} +import io.gearpump.cluster.scheduler.{Relaxation, Resource, ResourceAllocation, ResourceRequest} +import io.gearpump.cluster.worker.WorkerId +import io.gearpump.cluster.{AppDescription, AppMasterContext, TestUtil, UserConfig} +import io.gearpump.experiments.distributeservice.DistServiceAppMaster.{FileContainer, GetFileContainer} import io.gearpump.util.ActorSystemBooter.RegisterActorSystem import io.gearpump.util.ActorUtil -import org.scalatest.{BeforeAndAfter, Matchers, WordSpec} - -import scala.concurrent.duration._ -class DistServiceAppMasterSpec extends WordSpec with Matchers with BeforeAndAfter{ +class DistServiceAppMasterSpec extends WordSpec with Matchers with BeforeAndAfter { implicit val system = ActorSystem("AppMasterSpec", TestUtil.DEFAULT_CONFIG) val mockMaster = TestProbe()(system) val mockWorker1 = TestProbe()(system) @@ -45,37 +47,41 @@ class DistServiceAppMasterSpec extends WordSpec with Matchers with BeforeAndAfte val workerList = List(WorkerId(1, 0L), WorkerId(2, 0L), WorkerId(3, 0L)) val resource = Resource(1) val appJar = None - val appDescription = AppDescription("app0", classOf[DistServiceAppMaster].getName, UserConfig.empty) + val appDescription = AppDescription("app0", classOf[DistServiceAppMaster].getName, + UserConfig.empty) "DistService AppMaster" should { "responsable for service distributing" in { val appMasterInfo = AppMasterRuntimeInfo(appId, "appName", mockWorker1.ref) - val appMasterContext = AppMasterContext(appId, userName, resource, null, appJar, masterProxy, appMasterInfo) + val appMasterContext = AppMasterContext(appId, userName, resource, null, appJar, masterProxy, + appMasterInfo) TestActorRef[DistServiceAppMaster]( AppMasterRuntimeEnvironment.props(List(masterProxy.path), appDescription, appMasterContext)) - val registerAppMaster = mockMaster.receiveOne(15 seconds) + val registerAppMaster = mockMaster.receiveOne(15.seconds) assert(registerAppMaster.isInstanceOf[RegisterAppMaster]) val appMaster = registerAppMaster.asInstanceOf[RegisterAppMaster].appMaster mockMaster.reply(AppMasterRegistered(appId)) - //The DistributedShell AppMaster will ask for worker list + // The DistributedShell AppMaster will ask for worker list mockMaster.expectMsg(GetAllWorkers) mockMaster.reply(WorkerList(workerList)) - //After worker list is ready, DistributedShell AppMaster will request resouce on each worker + // After worker list is ready, DistributedShell AppMaster will request resouce on each worker workerList.foreach { workerId => - mockMaster.expectMsg(RequestResource(appId, ResourceRequest(Resource(1), workerId, relaxation = Relaxation.SPECIFICWORKER))) + mockMaster.expectMsg(RequestResource(appId, ResourceRequest(Resource(1), workerId, + relaxation = Relaxation.SPECIFICWORKER))) } - mockMaster.reply(ResourceAllocated(Array(ResourceAllocation(resource, mockWorker1.ref, WorkerId(1, 0L))))) + mockMaster.reply(ResourceAllocated(Array(ResourceAllocation(resource, mockWorker1.ref, + WorkerId(1, 0L))))) mockWorker1.expectMsgClass(classOf[LaunchExecutor]) mockWorker1.reply(RegisterActorSystem(ActorUtil.getSystemAddress(system).toString)) appMaster.tell(GetFileContainer, client.ref) - client.expectMsgClass(15 seconds, classOf[FileContainer]) + client.expectMsgClass(15.seconds, classOf[FileContainer]) } } after { - system.shutdown() - system.awaitTermination() + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } } diff --git a/examples/pagerank/src/main/scala/io/gearpump/experiments/pagerank/PageRankApplication.scala b/examples/pagerank/src/main/scala/io/gearpump/experiments/pagerank/PageRankApplication.scala index 2f9928c20..2e3709142 100644 --- a/examples/pagerank/src/main/scala/io/gearpump/experiments/pagerank/PageRankApplication.scala +++ b/examples/pagerank/src/main/scala/io/gearpump/experiments/pagerank/PageRankApplication.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,11 +18,12 @@ package io.gearpump.experiments.pagerank import akka.actor.ActorSystem -import io.gearpump.streaming.{StreamApplication, Processor} -import io.gearpump.streaming.appmaster.AppMaster + import io.gearpump.cluster.{Application, ApplicationMaster, UserConfig} -import PageRankApplication.NodeWithTaskId +import io.gearpump.experiments.pagerank.PageRankApplication.NodeWithTaskId import io.gearpump.partitioner.HashPartitioner +import io.gearpump.streaming.appmaster.AppMaster +import io.gearpump.streaming.{Processor, StreamApplication} import io.gearpump.util.Graph import io.gearpump.util.Graph.Node @@ -30,21 +31,19 @@ import io.gearpump.util.Graph.Node * * A simple and naive pagerank implementation. * - * We will continue to optimize this to able to run page rank of tens of millions of nodes - * * @param name name of the application * @param iteration max iteration count * @param delta decide the accuracy when the page rank example stops. * @param dag the page rank graph - * @tparam T */ -class PageRankApplication[T] (override val name : String, iteration: Int, delta: Double, dag: Graph[T, _]) +class PageRankApplication[T]( + override val name: String, iteration: Int, delta: Double, dag: Graph[T, _]) extends Application { override def appMaster: Class[_ <: ApplicationMaster] = classOf[AppMaster] override def userConfig(implicit system: ActorSystem): UserConfig = { - // map node with taskId + // Map node with taskId var taskId = 0 val pageRankDag = dag.mapVertex { node => val updatedNode = NodeWithTaskId(taskId, node) diff --git a/examples/pagerank/src/main/scala/io/gearpump/experiments/pagerank/PageRankController.scala b/examples/pagerank/src/main/scala/io/gearpump/experiments/pagerank/PageRankController.scala index 8771a408d..0fb689d3c 100644 --- a/examples/pagerank/src/main/scala/io/gearpump/experiments/pagerank/PageRankController.scala +++ b/examples/pagerank/src/main/scala/io/gearpump/experiments/pagerank/PageRankController.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,12 +18,14 @@ package io.gearpump.experiments.pagerank import akka.actor.Actor.Receive -import io.gearpump.streaming.task._ + import io.gearpump.cluster.UserConfig -import PageRankController.Tick -import PageRankWorker.LatestWeight +import io.gearpump.experiments.pagerank.PageRankController.Tick +import io.gearpump.experiments.pagerank.PageRankWorker.LatestWeight +import io.gearpump.streaming.task._ -class PageRankController (taskContext : TaskContext, conf: UserConfig) extends Task(taskContext, conf) { +class PageRankController(taskContext: TaskContext, conf: UserConfig) + extends Task(taskContext, conf) { val taskCount = conf.getInt(PageRankApplication.COUNT).get val iterationMax = conf.getInt(PageRankApplication.ITERATION).get @@ -37,11 +39,11 @@ class PageRankController (taskContext : TaskContext, conf: UserConfig) extends T var weights = Map.empty[TaskId, Double] var deltas = Map.empty[TaskId, Double] - override def onStart(startTime : StartTime) : Unit = { + override def onStart(startTime: StartTime): Unit = { output(Tick(tick), tasks: _*) } - private def output(msg: AnyRef, tasks: TaskId *): Unit = { + private def output(msg: AnyRef, tasks: TaskId*): Unit = { taskContext.asInstanceOf[TaskWrapper].outputUnManaged(msg, tasks: _*) } @@ -49,7 +51,7 @@ class PageRankController (taskContext : TaskContext, conf: UserConfig) extends T case LatestWeight(taskId, weight, replyTick) => if (this.tick == replyTick) { - deltas += taskId -> Math.abs(weight - weights.getOrElse(taskId, 0.0)) + deltas += taskId -> Math.abs(weight - weights.getOrElse(taskId, 0.0)) weights += taskId -> weight receivedWeightForCurrentTick += 1 if (receivedWeightForCurrentTick == taskCount) { @@ -66,7 +68,7 @@ class PageRankController (taskContext : TaskContext, conf: UserConfig) extends T } private def continueIteration: Boolean = { - (tick < iterationMax) && deltas.values.foldLeft(false) {(deltaExceed, value) => + (tick < iterationMax) && deltas.values.foldLeft(false) { (deltaExceed, value) => deltaExceed || value > delta } } diff --git a/examples/pagerank/src/main/scala/io/gearpump/experiments/pagerank/PageRankWorker.scala b/examples/pagerank/src/main/scala/io/gearpump/experiments/pagerank/PageRankWorker.scala index 87c30c2cf..9f38cd70a 100644 --- a/examples/pagerank/src/main/scala/io/gearpump/experiments/pagerank/PageRankWorker.scala +++ b/examples/pagerank/src/main/scala/io/gearpump/experiments/pagerank/PageRankWorker.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,17 +18,17 @@ package io.gearpump.experiments.pagerank import akka.actor.Actor.Receive -import io.gearpump.streaming.task.{Task, TaskContext, TaskId, TaskWrapper} + import io.gearpump.cluster.UserConfig -import PageRankApplication.NodeWithTaskId -import PageRankController.Tick -import PageRankWorker.{LatestWeight, UpdateWeight} +import io.gearpump.experiments.pagerank.PageRankApplication.NodeWithTaskId +import io.gearpump.experiments.pagerank.PageRankController.Tick +import io.gearpump.experiments.pagerank.PageRankWorker.{LatestWeight, UpdateWeight} +import io.gearpump.streaming.task.{Task, TaskContext, TaskId, TaskWrapper} import io.gearpump.util.Graph -class PageRankWorker(taskContext : TaskContext, conf: UserConfig) extends Task(taskContext, conf) { +class PageRankWorker(taskContext: TaskContext, conf: UserConfig) extends Task(taskContext, conf) { import taskContext.taskId - private var weight: Double = 1.0 private var upstreamWeights = Map.empty[TaskId, Double] @@ -41,14 +41,15 @@ class PageRankWorker(taskContext : TaskContext, conf: UserConfig) extends Task(t node.taskId == taskContext.taskId.index }.get - private val downstream = graph.outgoingEdgesOf(node).map(_._3).map(id => taskId.copy(index = id.taskId)).toSeq + private val downstream = graph.outgoingEdgesOf(node).map(_._3) + .map(id => taskId.copy(index = id.taskId)).toSeq private val upstreamCount = graph.incomingEdgesOf(node).map(_._1).length - LOG.info(s"downstream nodes: $downstream" ) + LOG.info(s"downstream nodes: $downstream") private var tick = 0 - private def output(msg: AnyRef, tasks: TaskId *): Unit = { + private def output(msg: AnyRef, tasks: TaskId*): Unit = { taskContext.asInstanceOf[TaskWrapper].outputUnManaged(msg, tasks: _*) } @@ -57,7 +58,7 @@ class PageRankWorker(taskContext : TaskContext, conf: UserConfig) extends Task(t this.tick = tick if (downstream.length == 0) { - // if there is no downstream, we will evenly distribute our page rank to + // If there is no downstream, we will evenly distribute our page rank to // every node in the graph val update = UpdateWeight(taskId, weight / taskCount) output(update, allTasks: _*) @@ -65,10 +66,10 @@ class PageRankWorker(taskContext : TaskContext, conf: UserConfig) extends Task(t val update = UpdateWeight(taskId, weight / downstream.length) output(update, downstream: _*) } - case update@ UpdateWeight(upstreamTaskId, weight) => + case update@UpdateWeight(upstreamTaskId, weight) => upstreamWeights += upstreamTaskId -> weight if (upstreamWeights.size == upstreamCount) { - val nextWeight = upstreamWeights.foldLeft(0.0) {(sum, item) => sum + item._2} + val nextWeight = upstreamWeights.foldLeft(0.0) { (sum, item) => sum + item._2 } this.upstreamWeights = Map.empty[TaskId, Double] this.weight = nextWeight output(LatestWeight(taskId, weight, tick), TaskId(0, 0)) @@ -76,7 +77,7 @@ class PageRankWorker(taskContext : TaskContext, conf: UserConfig) extends Task(t } } -object PageRankWorker{ +object PageRankWorker { case class UpdateWeight(taskId: TaskId, weight: Double) case class LatestWeight(taskId: TaskId, weight: Double, tick: Int) } \ No newline at end of file diff --git a/examples/pagerank/src/main/scala/io/gearpump/experiments/pagerank/example/PageRankExample.scala b/examples/pagerank/src/main/scala/io/gearpump/experiments/pagerank/example/PageRankExample.scala index 2fb9b8115..387797481 100644 --- a/examples/pagerank/src/main/scala/io/gearpump/experiments/pagerank/example/PageRankExample.scala +++ b/examples/pagerank/src/main/scala/io/gearpump/experiments/pagerank/example/PageRankExample.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,11 +17,12 @@ */ package io.gearpump.experiments.pagerank.example -import io.gearpump.experiments.pagerank.PageRankApplication import io.gearpump.cluster.client.ClientContext -import io.gearpump.util.{AkkaApp, Graph} +import io.gearpump.experiments.pagerank.PageRankApplication import io.gearpump.util.Graph.Node +import io.gearpump.util.{AkkaApp, Graph} +/** A very simple PageRank example, Cyclic graph is not supported */ object PageRankExample extends AkkaApp { val a = "a" @@ -29,7 +30,7 @@ object PageRankExample extends AkkaApp { val c = "c" val d = "d" - def help: Unit = Unit + def help(): Unit = Unit def main(akkaConf: Config, args: Array[String]): Unit = { val pageRankGraph = Graph(a ~> b, a ~> c, a ~> d, b ~> a, b ~> d, d ~> b, d ~> c, c ~> b) diff --git a/examples/streaming/complexdag/src/main/scala/io/gearpump/streaming/examples/complexdag/Dag.scala b/examples/streaming/complexdag/src/main/scala/io/gearpump/streaming/examples/complexdag/Dag.scala index a7e1f9231..b517126fb 100644 --- a/examples/streaming/complexdag/src/main/scala/io/gearpump/streaming/examples/complexdag/Dag.scala +++ b/examples/streaming/complexdag/src/main/scala/io/gearpump/streaming/examples/complexdag/Dag.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,57 +18,58 @@ package io.gearpump.streaming.examples.complexdag -import io.gearpump.streaming.{StreamApplication, Processor} -import io.gearpump.streaming.task.TaskContext +import org.slf4j.Logger + import io.gearpump.cluster.UserConfig import io.gearpump.cluster.client.ClientContext import io.gearpump.cluster.main.{ArgumentsParser, CLIOption, ParseResult} import io.gearpump.partitioner.HashPartitioner +import io.gearpump.streaming.task.TaskContext +import io.gearpump.streaming.{Processor, StreamApplication} import io.gearpump.util.Graph.{Node => GraphNode} import io.gearpump.util.{AkkaApp, Graph, LogUtil} -import org.slf4j.Logger -/* - digraph flow { - Source_0 -> Sink_0; - Source_0 -> Sink_1; - Source_0 -> Sink_2; - Source_0 -> Node_1; - Source_1 -> Node_0; - Node_0 -> Sink_3; - Node_1 -> Sink_3; - Node_1 -> Sink_4; - Node_1 -> Node_4; - Node_2 -> Node_3; - Node_1 -> Node_3; - Source_0 -> Node_2; - Source_0 -> Node_3; - Node_3 -> Sink_3; - Node_4 -> Sink_3; - Source_1 -> Sink_4; - } -*/ -case class Source_0(_context : TaskContext, _conf: UserConfig) extends Source(_context, _conf) -case class Source_1(_context : TaskContext, _conf: UserConfig) extends Source(_context, _conf) -case class Node_0(_context : TaskContext, _conf: UserConfig) extends Node(_context, _conf) -case class Node_1(_context : TaskContext, _conf: UserConfig) extends Node(_context, _conf) -case class Node_2(_context : TaskContext, _conf: UserConfig) extends Node(_context, _conf) -case class Node_3(_context : TaskContext, _conf: UserConfig) extends Node(_context, _conf) -case class Node_4(_context : TaskContext, _conf: UserConfig) extends Node(_context, _conf) -case class Sink_0(_context : TaskContext, _conf: UserConfig) extends Sink(_context, _conf) -case class Sink_1(_context : TaskContext, _conf: UserConfig) extends Sink(_context, _conf) -case class Sink_2(_context : TaskContext, _conf: UserConfig) extends Sink(_context, _conf) -case class Sink_3(_context : TaskContext, _conf: UserConfig) extends Sink(_context, _conf) -case class Sink_4(_context : TaskContext, _conf: UserConfig) extends Sink(_context, _conf) +case class Source_0(_context: TaskContext, _conf: UserConfig) extends Source(_context, _conf) +case class Source_1(_context: TaskContext, _conf: UserConfig) extends Source(_context, _conf) +case class Node_0(_context: TaskContext, _conf: UserConfig) extends Node(_context, _conf) +case class Node_1(_context: TaskContext, _conf: UserConfig) extends Node(_context, _conf) +case class Node_2(_context: TaskContext, _conf: UserConfig) extends Node(_context, _conf) +case class Node_3(_context: TaskContext, _conf: UserConfig) extends Node(_context, _conf) +case class Node_4(_context: TaskContext, _conf: UserConfig) extends Node(_context, _conf) +case class Sink_0(_context: TaskContext, _conf: UserConfig) extends Sink(_context, _conf) +case class Sink_1(_context: TaskContext, _conf: UserConfig) extends Sink(_context, _conf) +case class Sink_2(_context: TaskContext, _conf: UserConfig) extends Sink(_context, _conf) +case class Sink_3(_context: TaskContext, _conf: UserConfig) extends Sink(_context, _conf) +case class Sink_4(_context: TaskContext, _conf: UserConfig) extends Sink(_context, _conf) +/** + * digraph flow { + * Source_0 -> Sink_0; + * Source_0 -> Sink_1; + * Source_0 -> Sink_2; + * Source_0 -> Node_1; + * Source_1 -> Node_0; + * Node_0 -> Sink_3; + * Node_1 -> Sink_3; + * Node_1 -> Sink_4; + * Node_1 -> Node_4; + * Node_2 -> Node_3; + * Node_1 -> Node_3; + * Source_0 -> Node_2; + * Source_0 -> Node_3; + * Node_3 -> Sink_3; + * Node_4 -> Sink_3; + * Source_1 -> Sink_4; + * } + */ object Dag extends AkkaApp with ArgumentsParser { private val LOG: Logger = LogUtil.getLogger(getClass) val RUN_FOR_EVER = -1 override val options: Array[(String, CLIOption[Any])] = Array.empty - def application(config: ParseResult) : StreamApplication = { - + def application(config: ParseResult): StreamApplication = { + val source_0 = Processor[Source_0](1) val source_1 = Processor[Source_1](1) val node_0 = Processor[Node_0](1) diff --git a/examples/streaming/complexdag/src/main/scala/io/gearpump/streaming/examples/complexdag/Node.scala b/examples/streaming/complexdag/src/main/scala/io/gearpump/streaming/examples/complexdag/Node.scala index e8837ed69..dfad6c89d 100644 --- a/examples/streaming/complexdag/src/main/scala/io/gearpump/streaming/examples/complexdag/Node.scala +++ b/examples/streaming/complexdag/src/main/scala/io/gearpump/streaming/examples/complexdag/Node.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,16 +18,16 @@ package io.gearpump.streaming.examples.complexdag -import io.gearpump.streaming.task.{StartTime, Task, TaskContext} import io.gearpump.Message import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.task.{StartTime, Task, TaskContext} -class Node (taskContext : TaskContext, conf: UserConfig) extends Task(taskContext, conf) { +class Node(taskContext: TaskContext, conf: UserConfig) extends Task(taskContext, conf) { import taskContext.output - override def onStart(startTime : StartTime) : Unit = {} + override def onStart(startTime: StartTime): Unit = {} - override def onNext(msg : Message) : Unit = { + override def onNext(msg: Message): Unit = { val list = msg.msg.asInstanceOf[Vector[String]] output(new Message(list :+ getClass.getCanonicalName, System.currentTimeMillis())) } diff --git a/examples/streaming/complexdag/src/main/scala/io/gearpump/streaming/examples/complexdag/Sink.scala b/examples/streaming/complexdag/src/main/scala/io/gearpump/streaming/examples/complexdag/Sink.scala index adc311503..b091a17c7 100644 --- a/examples/streaming/complexdag/src/main/scala/io/gearpump/streaming/examples/complexdag/Sink.scala +++ b/examples/streaming/complexdag/src/main/scala/io/gearpump/streaming/examples/complexdag/Sink.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,11 +18,11 @@ package io.gearpump.streaming.examples.complexdag -import io.gearpump.streaming.task.{StartTime, Task, TaskContext} +import scala.collection.mutable + import io.gearpump.Message import io.gearpump.cluster.UserConfig - -import scala.collection.mutable +import io.gearpump.streaming.task.{StartTime, Task, TaskContext} class Sink(taskContext: TaskContext, conf: UserConfig) extends Task(taskContext, conf) { @@ -42,5 +42,4 @@ class Sink(taskContext: TaskContext, conf: UserConfig) extends Task(taskContext, case _ => } } - } diff --git a/examples/streaming/complexdag/src/main/scala/io/gearpump/streaming/examples/complexdag/Source.scala b/examples/streaming/complexdag/src/main/scala/io/gearpump/streaming/examples/complexdag/Source.scala index 80cd8d267..df656acde 100644 --- a/examples/streaming/complexdag/src/main/scala/io/gearpump/streaming/examples/complexdag/Source.scala +++ b/examples/streaming/complexdag/src/main/scala/io/gearpump/streaming/examples/complexdag/Source.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,12 +18,12 @@ package io.gearpump.streaming.examples.complexdag -import io.gearpump.streaming.task.{StartTime, Task, TaskContext} import io.gearpump.Message import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.task.{StartTime, Task, TaskContext} class Source(taskContext: TaskContext, conf: UserConfig) extends Task(taskContext, conf) { - import taskContext.{output, self} + import taskContext.output override def onStart(startTime: StartTime): Unit = { self ! Message("start") diff --git a/examples/streaming/complexdag/src/test/scala/io/gearpump/streaming/examples/complexdag/DagSpec.scala b/examples/streaming/complexdag/src/test/scala/io/gearpump/streaming/examples/complexdag/DagSpec.scala index 02a1017a3..b142d8d9c 100644 --- a/examples/streaming/complexdag/src/test/scala/io/gearpump/streaming/examples/complexdag/DagSpec.scala +++ b/examples/streaming/complexdag/src/test/scala/io/gearpump/streaming/examples/complexdag/DagSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,16 +18,18 @@ package io.gearpump.streaming.examples.complexdag -import io.gearpump.cluster.ClientToMaster.SubmitApplication -import io.gearpump.cluster.MasterToClient.SubmitApplicationResult -import io.gearpump.cluster.{TestUtil, MasterHarness} -import io.gearpump.util.Util +import scala.concurrent.Future +import scala.util.Success + import org.scalatest._ import org.scalatest.prop.PropertyChecks -import scala.util.Success -import scala.concurrent.Future -class DagSpec extends PropSpec with PropertyChecks with Matchers with BeforeAndAfterAll with MasterHarness { +import io.gearpump.cluster.ClientToMaster.SubmitApplication +import io.gearpump.cluster.MasterToClient.SubmitApplicationResult +import io.gearpump.cluster.{MasterHarness, TestUtil} + +class DagSpec extends PropSpec with PropertyChecks + with Matchers with BeforeAndAfterAll with MasterHarness { override def beforeAll { startActorSystem() @@ -37,7 +39,7 @@ class DagSpec extends PropSpec with PropertyChecks with Matchers with BeforeAndA shutdownActorSystem() } - override def config = TestUtil.DEFAULT_CONFIG + protected override def config = TestUtil.DEFAULT_CONFIG property("Dag should succeed to submit application with required arguments") { val requiredArgs = Array.empty[String] @@ -45,7 +47,9 @@ class DagSpec extends PropSpec with PropertyChecks with Matchers with BeforeAndA val masterReceiver = createMockMaster() val args = requiredArgs - Future{Dag.main(masterConfig, args)} + Future { + Dag.main(masterConfig, args) + } masterReceiver.expectMsgType[SubmitApplication](PROCESS_BOOT_TIME) masterReceiver.reply(SubmitApplicationResult(Success(0))) } diff --git a/examples/streaming/complexdag/src/test/scala/io/gearpump/streaming/examples/complexdag/NodeSpec.scala b/examples/streaming/complexdag/src/test/scala/io/gearpump/streaming/examples/complexdag/NodeSpec.scala index d1516513b..35c5824e4 100644 --- a/examples/streaming/complexdag/src/test/scala/io/gearpump/streaming/examples/complexdag/NodeSpec.scala +++ b/examples/streaming/complexdag/src/test/scala/io/gearpump/streaming/examples/complexdag/NodeSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,25 +17,25 @@ */ package io.gearpump.streaming.examples.complexdag -import io.gearpump.streaming.MockUtil -import io.gearpump.Message -import io.gearpump.cluster.UserConfig -import MockUtil._ -import org.mockito.ArgumentMatcher -import org.mockito.Matchers._ import org.mockito.Mockito._ import org.scalatest.prop.PropertyChecks import org.scalatest.{BeforeAndAfter, Matchers, PropSpec} +import io.gearpump.Message +import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.MockUtil +import io.gearpump.streaming.MockUtil._ + class NodeSpec extends PropSpec with PropertyChecks with Matchers with BeforeAndAfter { val context = MockUtil.mockTaskContext val node = new Node(context, UserConfig.empty) - property("Node should send a Vector[String](classOf[Node].getCanonicalName, classOf[Node].getCanonicalName"){ + property("Node should send a Vector[String](classOf[Node].getCanonicalName, " + + "classOf[Node].getCanonicalName") { val list = Vector(classOf[Node].getCanonicalName) - val expected = Vector(classOf[Node].getCanonicalName,classOf[Node].getCanonicalName) + val expected = Vector(classOf[Node].getCanonicalName, classOf[Node].getCanonicalName) node.onNext(Message(list)) verify(context).output(argMatch[Message](_.msg == expected)) } diff --git a/examples/streaming/complexdag/src/test/scala/io/gearpump/streaming/examples/complexdag/SinkSpec.scala b/examples/streaming/complexdag/src/test/scala/io/gearpump/streaming/examples/complexdag/SinkSpec.scala index b26f639a7..341f6c664 100644 --- a/examples/streaming/complexdag/src/test/scala/io/gearpump/streaming/examples/complexdag/SinkSpec.scala +++ b/examples/streaming/complexdag/src/test/scala/io/gearpump/streaming/examples/complexdag/SinkSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,19 +17,21 @@ */ package io.gearpump.streaming.examples.complexdag -import io.gearpump.streaming.MockUtil -import io.gearpump.Message -import io.gearpump.cluster.UserConfig import org.scalatest.prop.PropertyChecks import org.scalatest.{BeforeAndAfter, Matchers, PropSpec} +import io.gearpump.Message +import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.MockUtil + class SinkSpec extends PropSpec with PropertyChecks with Matchers with BeforeAndAfter { val context = MockUtil.mockTaskContext val sink = new Sink(context, UserConfig.empty) - property("Sink should send a Vector[String](classOf[Sink].getCanonicalName, classOf[Sink].getCanonicalName"){ + property("Sink should send a Vector[String](classOf[Sink].getCanonicalName, " + + "classOf[Sink].getCanonicalName") { val list = Vector(classOf[Sink].getCanonicalName) val expected = Vector(classOf[Sink].getCanonicalName, classOf[Sink].getCanonicalName) sink.onNext(Message(list)) diff --git a/examples/streaming/complexdag/src/test/scala/io/gearpump/streaming/examples/complexdag/SourceSpec.scala b/examples/streaming/complexdag/src/test/scala/io/gearpump/streaming/examples/complexdag/SourceSpec.scala index 21ae6abf2..faa7aa7e1 100644 --- a/examples/streaming/complexdag/src/test/scala/io/gearpump/streaming/examples/complexdag/SourceSpec.scala +++ b/examples/streaming/complexdag/src/test/scala/io/gearpump/streaming/examples/complexdag/SourceSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,15 +18,14 @@ package io.gearpump.streaming.examples.complexdag import akka.actor.ActorSystem -import io.gearpump.streaming.MockUtil -import io.gearpump.Message -import io.gearpump.cluster.{TestUtil, UserConfig} -import MockUtil._ -import org.mockito.ArgumentMatcher -import org.mockito.Matchers._ import org.mockito.Mockito._ import org.scalatest.{Matchers, WordSpec} +import io.gearpump.Message +import io.gearpump.cluster.{TestUtil, UserConfig} +import io.gearpump.streaming.MockUtil +import io.gearpump.streaming.MockUtil._ + class SourceSpec extends WordSpec with Matchers { "Source" should { diff --git a/examples/streaming/fsio/src/main/scala/io/gearpump/streaming/examples/fsio/HadoopConfig.scala b/examples/streaming/fsio/src/main/scala/io/gearpump/streaming/examples/fsio/HadoopConfig.scala index c8c1135a5..3b53c9c84 100644 --- a/examples/streaming/fsio/src/main/scala/io/gearpump/streaming/examples/fsio/HadoopConfig.scala +++ b/examples/streaming/fsio/src/main/scala/io/gearpump/streaming/examples/fsio/HadoopConfig.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,19 +18,22 @@ package io.gearpump.streaming.examples.fsio import java.io.{ByteArrayInputStream, ByteArrayOutputStream, DataInputStream, DataOutputStream} +import scala.language.implicitConversions + +import org.apache.hadoop.conf.Configuration import io.gearpump.cluster.UserConfig import io.gearpump.util.Constants._ -import org.apache.hadoop.conf.Configuration -import scala.language.implicitConversions +class HadoopConfig(config: UserConfig) { -class HadoopConfig(config: UserConfig) { + def withHadoopConf(conf: Configuration): UserConfig = { + config.withBytes(HADOOP_CONF, serializeHadoopConf(conf)) + } - def withHadoopConf(conf : Configuration) : UserConfig = config.withBytes(HADOOP_CONF, serializeHadoopConf(conf)) - def hadoopConf : Configuration = deserializeHadoopConf(config.getBytes(HADOOP_CONF).get) + def hadoopConf: Configuration = deserializeHadoopConf(config.getBytes(HADOOP_CONF).get) - private def serializeHadoopConf(conf: Configuration) : Array[Byte] = { + private def serializeHadoopConf(conf: Configuration): Array[Byte] = { val out = new ByteArrayOutputStream() val dataOut = new DataOutputStream(out) conf.write(dataOut) @@ -38,10 +41,10 @@ class HadoopConfig(config: UserConfig) { out.toByteArray } - private def deserializeHadoopConf(bytes: Array[Byte]) : Configuration = { + private def deserializeHadoopConf(bytes: Array[Byte]): Configuration = { val in = new ByteArrayInputStream(bytes) val dataIn = new DataInputStream(in) - val result= new Configuration() + val result = new Configuration() result.readFields(dataIn) dataIn.close() result @@ -49,8 +52,8 @@ class HadoopConfig(config: UserConfig) { } object HadoopConfig { - def empty = new HadoopConfig(UserConfig.empty) - def apply(config: UserConfig) = new HadoopConfig(config) + def empty: HadoopConfig = new HadoopConfig(UserConfig.empty) + def apply(config: UserConfig): HadoopConfig = new HadoopConfig(config) implicit def userConfigToHadoopConfig(userConf: UserConfig): HadoopConfig = { HadoopConfig(userConf) diff --git a/examples/streaming/fsio/src/main/scala/io/gearpump/streaming/examples/fsio/SeqFileStreamProcessor.scala b/examples/streaming/fsio/src/main/scala/io/gearpump/streaming/examples/fsio/SeqFileStreamProcessor.scala index 808f3b180..13fc3f9a7 100644 --- a/examples/streaming/fsio/src/main/scala/io/gearpump/streaming/examples/fsio/SeqFileStreamProcessor.scala +++ b/examples/streaming/fsio/src/main/scala/io/gearpump/streaming/examples/fsio/SeqFileStreamProcessor.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,20 +19,21 @@ package io.gearpump.streaming.examples.fsio import java.io.File import java.util.concurrent.TimeUnit +import scala.concurrent.duration.FiniteDuration import akka.actor.Cancellable -import io.gearpump.streaming.task.{StartTime, Task, TaskContext} -import io.gearpump.Message -import io.gearpump.cluster.UserConfig -import SeqFileStreamProcessor._ -import HadoopConfig._ import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.hadoop.io.SequenceFile._ import org.apache.hadoop.io.{SequenceFile, Text} -import scala.concurrent.duration.FiniteDuration +import io.gearpump.Message +import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.examples.fsio.HadoopConfig._ +import io.gearpump.streaming.examples.fsio.SeqFileStreamProcessor._ +import io.gearpump.streaming.task.{StartTime, Task, TaskContext} -class SeqFileStreamProcessor(taskContext : TaskContext, config: UserConfig) extends Task(taskContext, config){ +class SeqFileStreamProcessor(taskContext: TaskContext, config: UserConfig) + extends Task(taskContext, config) { import taskContext.taskId @@ -43,16 +44,17 @@ class SeqFileStreamProcessor(taskContext : TaskContext, config: UserConfig) exte val value = new Text() val hadoopConf = config.hadoopConf - private var msgCount : Long = 0 - private var snapShotKVCount : Long = 0 - private var snapShotTime : Long = 0 + private var msgCount: Long = 0 + private var snapShotKVCount: Long = 0 + private var snapShotTime: Long = 0 private var scheduler: Cancellable = null - override def onStart(startTime : StartTime) = { + override def onStart(startTime: StartTime): Unit = { val fs = FileSystem.get(hadoopConf) fs.deleteOnExit(outputPath) - writer = SequenceFile.createWriter(hadoopConf, Writer.file(outputPath), Writer.keyClass(textClass), Writer.valueClass(textClass)) + writer = SequenceFile.createWriter(hadoopConf, Writer.file(outputPath), + Writer.keyClass(textClass), Writer.valueClass(textClass)) scheduler = taskContext.schedule(new FiniteDuration(5, TimeUnit.SECONDS), new FiniteDuration(5, TimeUnit.SECONDS))(reportStatus()) @@ -62,7 +64,7 @@ class SeqFileStreamProcessor(taskContext : TaskContext, config: UserConfig) exte override def onNext(msg: Message): Unit = { val kv = msg.msg.asInstanceOf[String].split("\\+\\+") - if(kv.length >= 2) { + if (kv.length >= 2) { key.set(kv(0)) value.set(kv(1)) writer.append(key, value) @@ -70,7 +72,7 @@ class SeqFileStreamProcessor(taskContext : TaskContext, config: UserConfig) exte msgCount += 1 } - override def onStop(): Unit ={ + override def onStop(): Unit = { if (scheduler != null) { scheduler.cancel() } @@ -78,14 +80,17 @@ class SeqFileStreamProcessor(taskContext : TaskContext, config: UserConfig) exte LOG.info("sequence file bolt stopped") } - def reportStatus() = { - val current : Long = System.currentTimeMillis() - LOG.info(s"Task $taskId Throughput: ${(msgCount - snapShotKVCount, (current - snapShotTime) / 1000)} (KVPairs, second)") + private def reportStatus() = { + val current: Long = System.currentTimeMillis() + LOG.info(s"Task $taskId Throughput: ${ + (msgCount - snapShotKVCount, + (current - snapShotTime) / 1000) + } (KVPairs, second)") snapShotKVCount = msgCount snapShotTime = current } } -object SeqFileStreamProcessor{ +object SeqFileStreamProcessor { val OUTPUT_PATH = "outputpath" } \ No newline at end of file diff --git a/examples/streaming/fsio/src/main/scala/io/gearpump/streaming/examples/fsio/SeqFileStreamProducer.scala b/examples/streaming/fsio/src/main/scala/io/gearpump/streaming/examples/fsio/SeqFileStreamProducer.scala index 5d32f7430..f9b0d2253 100644 --- a/examples/streaming/fsio/src/main/scala/io/gearpump/streaming/examples/fsio/SeqFileStreamProducer.scala +++ b/examples/streaming/fsio/src/main/scala/io/gearpump/streaming/examples/fsio/SeqFileStreamProducer.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,18 +17,20 @@ */ package io.gearpump.streaming.examples.fsio -import io.gearpump.streaming.task.{StartTime, Task, TaskContext} -import io.gearpump.Message -import io.gearpump.cluster.UserConfig -import SeqFileStreamProducer._ -import HadoopConfig._ import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.hadoop.io.SequenceFile._ import org.apache.hadoop.io.{SequenceFile, Text} -class SeqFileStreamProducer(taskContext : TaskContext, config: UserConfig) extends Task(taskContext, config){ +import io.gearpump.Message +import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.examples.fsio.HadoopConfig._ +import io.gearpump.streaming.examples.fsio.SeqFileStreamProducer._ +import io.gearpump.streaming.task.{StartTime, Task, TaskContext} + +class SeqFileStreamProducer(taskContext: TaskContext, config: UserConfig) + extends Task(taskContext, config) { - import taskContext.{output, self} + import taskContext.output val value = new Text() val key = new Text() @@ -37,14 +39,14 @@ class SeqFileStreamProducer(taskContext : TaskContext, config: UserConfig) exten val fs = FileSystem.get(hadoopConf) val inputPath = new Path(config.getString(INPUT_PATH).get) - override def onStart(startTime : StartTime) = { + override def onStart(startTime: StartTime): Unit = { reader = new SequenceFile.Reader(hadoopConf, Reader.file(inputPath)) self ! Start LOG.info("sequence file spout initiated") } - override def onNext(msg: Message) = { - if(reader.next(key, value)){ + override def onNext(msg: Message): Unit = { + if (reader.next(key, value)) { output(Message(key + "++" + value)) } else { reader.close() @@ -53,13 +55,13 @@ class SeqFileStreamProducer(taskContext : TaskContext, config: UserConfig) exten self ! Continue } - override def onStop(): Unit ={ + override def onStop(): Unit = { reader.close() } } -object SeqFileStreamProducer{ - def INPUT_PATH = "inputpath" +object SeqFileStreamProducer { + def INPUT_PATH: String = "inputpath" val Start = Message("start") val Continue = Message("continue") diff --git a/examples/streaming/fsio/src/main/scala/io/gearpump/streaming/examples/fsio/SequenceFileIO.scala b/examples/streaming/fsio/src/main/scala/io/gearpump/streaming/examples/fsio/SequenceFileIO.scala index c752bba5f..7272f2b9d 100644 --- a/examples/streaming/fsio/src/main/scala/io/gearpump/streaming/examples/fsio/SequenceFileIO.scala +++ b/examples/streaming/fsio/src/main/scala/io/gearpump/streaming/examples/fsio/SequenceFileIO.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,39 +17,44 @@ */ package io.gearpump.streaming.examples.fsio -import io.gearpump.streaming.{StreamApplication, Processor} +import org.apache.hadoop.conf.Configuration +import org.slf4j.Logger + import io.gearpump.cluster.UserConfig import io.gearpump.cluster.client.ClientContext import io.gearpump.cluster.main.{ArgumentsParser, CLIOption, ParseResult} import io.gearpump.partitioner.ShufflePartitioner +import io.gearpump.streaming.examples.fsio.HadoopConfig._ +import io.gearpump.streaming.{Processor, StreamApplication} import io.gearpump.util.Graph._ -import HadoopConfig._ import io.gearpump.util.{AkkaApp, Graph, LogUtil} -import org.apache.hadoop.conf.Configuration -import org.slf4j.Logger object SequenceFileIO extends AkkaApp with ArgumentsParser { private val LOG: Logger = LogUtil.getLogger(getClass) override val options: Array[(String, CLIOption[Any])] = Array( - "source"-> CLIOption[Int]("", required = false, defaultValue = Some(1)), - "sink"-> CLIOption[Int]("", required = false, defaultValue = Some(1)), - "input"-> CLIOption[String]("", required = true), - "output"-> CLIOption[String]("", required = true) + "source" -> CLIOption[Int]("", required = false, + defaultValue = Some(1)), + "sink" -> CLIOption[Int]("", required = false, + defaultValue = Some(1)), + "input" -> CLIOption[String]("", required = true), + "output" -> CLIOption[String]("", required = true) ) - def application(config: ParseResult) : StreamApplication = { + def application(config: ParseResult): StreamApplication = { val spoutNum = config.getInt("source") val boltNum = config.getInt("sink") val input = config.getString("input") val output = config.getString("output") - val appConfig = UserConfig.empty.withString(SeqFileStreamProducer.INPUT_PATH, input).withString(SeqFileStreamProcessor.OUTPUT_PATH, output) + val appConfig = UserConfig.empty.withString(SeqFileStreamProducer.INPUT_PATH, input) + .withString(SeqFileStreamProcessor.OUTPUT_PATH, output) val hadoopConfig = appConfig.withHadoopConf(new Configuration()) val partitioner = new ShufflePartitioner() val streamProducer = Processor[SeqFileStreamProducer](spoutNum) val streamProcessor = Processor[SeqFileStreamProcessor](boltNum) - val app = StreamApplication("SequenceFileIO", Graph(streamProducer ~ partitioner ~> streamProcessor), hadoopConfig) + val app = StreamApplication("SequenceFileIO", + Graph(streamProducer ~ partitioner ~> streamProcessor), hadoopConfig) app } diff --git a/examples/streaming/fsio/src/test/scala/io/gearpump/streaming/examples/fsio/HadoopConfigSpec.scala b/examples/streaming/fsio/src/test/scala/io/gearpump/streaming/examples/fsio/HadoopConfigSpec.scala index 64ba69731..e5dbe0ba4 100644 --- a/examples/streaming/fsio/src/test/scala/io/gearpump/streaming/examples/fsio/HadoopConfigSpec.scala +++ b/examples/streaming/fsio/src/test/scala/io/gearpump/streaming/examples/fsio/HadoopConfigSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,10 +17,11 @@ */ package io.gearpump.streaming.examples.fsio -import io.gearpump.cluster.UserConfig import org.apache.hadoop.conf.Configuration import org.scalatest.{Matchers, WordSpec} +import io.gearpump.cluster.UserConfig + class HadoopConfigSpec extends WordSpec with Matchers { "HadoopConfig" should { @@ -32,7 +33,7 @@ class HadoopConfigSpec extends WordSpec with Matchers { val user = UserConfig.empty - import HadoopConfig._ + import io.gearpump.streaming.examples.fsio.HadoopConfig._ assert(user.withHadoopConf(hadoopConf).hadoopConf.get(key) == value) } } diff --git a/examples/streaming/fsio/src/test/scala/io/gearpump/streaming/examples/fsio/SeqFileStreamProcessorSpec.scala b/examples/streaming/fsio/src/test/scala/io/gearpump/streaming/examples/fsio/SeqFileStreamProcessorSpec.scala index 99c7113e9..bb0d26b1e 100644 --- a/examples/streaming/fsio/src/test/scala/io/gearpump/streaming/examples/fsio/SeqFileStreamProcessorSpec.scala +++ b/examples/streaming/fsio/src/test/scala/io/gearpump/streaming/examples/fsio/SeqFileStreamProcessorSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,15 +18,10 @@ package io.gearpump.streaming.examples.fsio import java.io.File +import scala.collection.mutable.ArrayBuffer import akka.actor.ActorSystem import akka.testkit.TestProbe -import io.gearpump.streaming.{Processor, MockUtil} -import io.gearpump.streaming.task.{StartTime, TaskId} -import io.gearpump.Message -import io.gearpump.cluster.{TestUtil, UserConfig} -import io.gearpump.streaming.Processor -import io.gearpump.streaming.task.StartTime import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.hadoop.io.SequenceFile.Reader @@ -36,8 +31,13 @@ import org.scalacheck.Gen import org.scalatest.prop.PropertyChecks import org.scalatest.{BeforeAndAfter, Matchers, PropSpec} -import scala.collection.mutable.ArrayBuffer -class SeqFileStreamProcessorSpec extends PropSpec with PropertyChecks with Matchers with BeforeAndAfter { +import io.gearpump.Message +import io.gearpump.cluster.{TestUtil, UserConfig} +import io.gearpump.streaming.task.{StartTime, TaskId} +import io.gearpump.streaming.{MockUtil, Processor} +class SeqFileStreamProcessorSpec + extends PropSpec with PropertyChecks with Matchers with BeforeAndAfter { + val kvPairs = new ArrayBuffer[(String, String)] val outputDirectory = "SeqFileStreamProcessor_Test" val sequenceFilePath = new Path(outputDirectory + File.separator + TaskId(0, 0)) @@ -56,7 +56,8 @@ class SeqFileStreamProcessorSpec extends PropSpec with PropertyChecks with Match implicit val system1 = ActorSystem("SeqFileStreamProcessor", TestUtil.DEFAULT_CONFIG) val system2 = ActorSystem("Reporter", TestUtil.DEFAULT_CONFIG) val watcher = TestProbe()(system1) - val conf = HadoopConfig(UserConfig.empty.withString(SeqFileStreamProcessor.OUTPUT_PATH, outputDirectory)).withHadoopConf(new Configuration()) + val conf = HadoopConfig(UserConfig.empty.withString(SeqFileStreamProcessor.OUTPUT_PATH, + outputDirectory)).withHadoopConf(new Configuration()) val context = MockUtil.mockTaskContext val processorDescription = @@ -80,7 +81,7 @@ class SeqFileStreamProcessorSpec extends PropSpec with PropertyChecks with Match val reader = new SequenceFile.Reader(hadoopConf, Reader.file(sequenceFilePath)) kvPairs.foreach { kv => val (key, value) = kv - if(value.length > 0 && reader.next(_key, _value)) { + if (value.length > 0 && reader.next(_key, _value)) { assert(_key.toString == key && _value.toString == value) } } @@ -90,4 +91,4 @@ class SeqFileStreamProcessorSpec extends PropSpec with PropertyChecks with Match after { fs.deleteOnExit(new Path(outputDirectory)) } -} +} \ No newline at end of file diff --git a/examples/streaming/fsio/src/test/scala/io/gearpump/streaming/examples/fsio/SeqFileStreamProducerSpec.scala b/examples/streaming/fsio/src/test/scala/io/gearpump/streaming/examples/fsio/SeqFileStreamProducerSpec.scala index cb6f55370..04dafa74a 100644 --- a/examples/streaming/fsio/src/test/scala/io/gearpump/streaming/examples/fsio/SeqFileStreamProducerSpec.scala +++ b/examples/streaming/fsio/src/test/scala/io/gearpump/streaming/examples/fsio/SeqFileStreamProducerSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,26 +17,26 @@ */ package io.gearpump.streaming.examples.fsio -import io.gearpump.streaming.MockUtil -import io.gearpump.streaming.examples.fsio.HadoopConfig -import io.gearpump.streaming.task.StartTime -import io.gearpump.Message -import io.gearpump.cluster.UserConfig -import MockUtil._ +import scala.collection.mutable.ArrayBuffer + import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.hadoop.io.SequenceFile.Writer import org.apache.hadoop.io.{SequenceFile, Text} -import org.mockito.ArgumentMatcher -import org.mockito.Matchers._ import org.mockito.Mockito._ import org.scalacheck.Gen import org.scalatest.prop.PropertyChecks import org.scalatest.{BeforeAndAfter, Matchers, PropSpec} -import scala.collection.mutable.ArrayBuffer +import io.gearpump.Message +import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.MockUtil +import io.gearpump.streaming.MockUtil._ +import io.gearpump.streaming.task.StartTime + +class SeqFileStreamProducerSpec + extends PropSpec with PropertyChecks with Matchers with BeforeAndAfter { -class SeqFileStreamProducerSpec extends PropSpec with PropertyChecks with Matchers with BeforeAndAfter{ val kvPairs = new ArrayBuffer[(String, String)] val inputFile = "SeqFileStreamProducer_Test" val sequenceFilePath = new Path(inputFile) @@ -53,7 +53,8 @@ class SeqFileStreamProducerSpec extends PropSpec with PropertyChecks with Matche before { fs.deleteOnExit(sequenceFilePath) - val writer = SequenceFile.createWriter(hadoopConf, Writer.file(sequenceFilePath), Writer.keyClass(textClass), Writer.valueClass(textClass)) + val writer = SequenceFile.createWriter(hadoopConf, Writer.file(sequenceFilePath), + Writer.keyClass(textClass), Writer.valueClass(textClass)) forAll(kvGenerator) { kv => _key.set(kv._1) _value.set(kv._2) @@ -63,9 +64,11 @@ class SeqFileStreamProducerSpec extends PropSpec with PropertyChecks with Matche writer.close() } - property("SeqFileStreamProducer should read the key-value pairs from a sequence file and deliver them") { + property("SeqFileStreamProducer should read the key-value pairs from " + + "a sequence file and deliver them") { - val conf = HadoopConfig(UserConfig.empty.withString(SeqFileStreamProducer.INPUT_PATH, inputFile)).withHadoopConf(new Configuration()) + val conf = HadoopConfig(UserConfig.empty.withString(SeqFileStreamProducer.INPUT_PATH, + inputFile)).withHadoopConf(new Configuration()) val context = MockUtil.mockTaskContext @@ -74,7 +77,8 @@ class SeqFileStreamProducerSpec extends PropSpec with PropertyChecks with Matche producer.onNext(Message("start")) val expected = kvPairs.map(kv => kv._1 + "++" + kv._2).toSet - verify(context).output(argMatch[Message](msg => expected.contains(msg.msg.asInstanceOf[String]))) + verify(context).output(argMatch[Message](msg => + expected.contains(msg.msg.asInstanceOf[String]))) } after { diff --git a/examples/streaming/fsio/src/test/scala/io/gearpump/streaming/examples/fsio/SequenceFileIOSpec.scala b/examples/streaming/fsio/src/test/scala/io/gearpump/streaming/examples/fsio/SequenceFileIOSpec.scala index f10227da3..efb5e4462 100644 --- a/examples/streaming/fsio/src/test/scala/io/gearpump/streaming/examples/fsio/SequenceFileIOSpec.scala +++ b/examples/streaming/fsio/src/test/scala/io/gearpump/streaming/examples/fsio/SequenceFileIOSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,17 +18,20 @@ package io.gearpump.streaming.examples.fsio +import scala.concurrent.Future +import scala.util.{Success, Try} + +import com.typesafe.config.Config +import org.scalatest.prop.PropertyChecks +import org.scalatest.{BeforeAndAfterAll, Matchers, PropSpec} + import io.gearpump.cluster.ClientToMaster.SubmitApplication import io.gearpump.cluster.MasterToClient.SubmitApplicationResult import io.gearpump.cluster.{MasterHarness, TestUtil} -import io.gearpump.util.Util -import org.scalatest.prop.PropertyChecks -import org.scalatest.{BeforeAndAfterAll, Matchers, PropSpec} -import scala.util.{Success, Try} -import scala.concurrent.Future +class SequenceFileIOSpec + extends PropSpec with PropertyChecks with Matchers with BeforeAndAfterAll with MasterHarness { -class SequenceFileIOSpec extends PropSpec with PropertyChecks with Matchers with BeforeAndAfterAll with MasterHarness { override def beforeAll { startActorSystem() } @@ -37,7 +40,7 @@ class SequenceFileIOSpec extends PropSpec with PropertyChecks with Matchers with shutdownActorSystem() } - override def config = TestUtil.DEFAULT_CONFIG + override def config: Config = TestUtil.DEFAULT_CONFIG property("SequenceFileIO should succeed to submit application with required arguments") { val requiredArgs = Array( @@ -58,7 +61,9 @@ class SequenceFileIOSpec extends PropSpec with PropertyChecks with Matchers with forAll(validArgs) { (requiredArgs: Array[String], optionalArgs: Array[String]) => val args = requiredArgs ++ optionalArgs - Future {SequenceFileIO.main(masterConfig, args)} + Future { + SequenceFileIO.main(masterConfig, args) + } masterReceiver.expectMsgType[SubmitApplication](PROCESS_BOOT_TIME) masterReceiver.reply(SubmitApplicationResult(Success(0))) } @@ -75,5 +80,4 @@ class SequenceFileIOSpec extends PropSpec with PropertyChecks with Matchers with assert(Try(SequenceFileIO.main(args)).isFailure, "missing required arguments, print usage") } } - } diff --git a/examples/streaming/kafka/src/main/scala/io/gearpump/streaming/examples/kafka/KafkaReadWrite.scala b/examples/streaming/kafka/src/main/scala/io/gearpump/streaming/examples/kafka/KafkaReadWrite.scala index b4d9a0402..35b659421 100644 --- a/examples/streaming/kafka/src/main/scala/io/gearpump/streaming/examples/kafka/KafkaReadWrite.scala +++ b/examples/streaming/kafka/src/main/scala/io/gearpump/streaming/examples/kafka/KafkaReadWrite.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,6 +19,8 @@ package io.gearpump.streaming.examples.kafka import akka.actor.ActorSystem +import org.slf4j.Logger + import io.gearpump.cluster.UserConfig import io.gearpump.cluster.client.ClientContext import io.gearpump.cluster.main.{ArgumentsParser, CLIOption, ParseResult} @@ -29,21 +31,26 @@ import io.gearpump.streaming.sink.DataSinkProcessor import io.gearpump.streaming.source.DataSourceProcessor import io.gearpump.util.Graph._ import io.gearpump.util.{AkkaApp, Graph, LogUtil} -import org.slf4j.Logger object KafkaReadWrite extends AkkaApp with ArgumentsParser { private val LOG: Logger = LogUtil.getLogger(getClass) override val options: Array[(String, CLIOption[Any])] = Array( - "source" -> CLIOption[Int]("", required = false, defaultValue = Some(1)), - "sink" -> CLIOption[Int]("", required = false, defaultValue = Some(1)), - "zookeeperConnect" -> CLIOption[String]("", required = false, defaultValue = Some("localhost:2181")), - "brokerList" -> CLIOption[String]("", required = false, defaultValue = Some("localhost:9092")), - "sourceTopic" -> CLIOption[String]("", required = false, defaultValue = Some("topic1")), - "sinkTopic" -> CLIOption[String]("", required = false, defaultValue = Some("topic2")) + "source" -> CLIOption[Int]("", required = false, + defaultValue = Some(1)), + "sink" -> CLIOption[Int]("", required = false, + defaultValue = Some(1)), + "zookeeperConnect" -> CLIOption[String]("", required = false, + defaultValue = Some("localhost:2181")), + "brokerList" -> CLIOption[String]("", required = false, + defaultValue = Some("localhost:9092")), + "sourceTopic" -> CLIOption[String]("", required = false, + defaultValue = Some("topic1")), + "sinkTopic" -> CLIOption[String]("", required = false, + defaultValue = Some("topic2")) ) - def application(config: ParseResult, system: ActorSystem) : StreamApplication = { + def application(config: ParseResult, system: ActorSystem): StreamApplication = { implicit val actorSystem = system val sourceNum = config.getInt("source") val sinkNum = config.getInt("sink") @@ -59,7 +66,7 @@ object KafkaReadWrite extends AkkaApp with ArgumentsParser { val sink = new KafkaSink(sinkTopic, brokerList) val sinkProcessor = DataSinkProcessor(sink, sinkNum) val partitioner = new ShufflePartitioner - val computation = sourceProcessor ~ partitioner ~> sinkProcessor + val computation = sourceProcessor ~ partitioner ~> sinkProcessor val app = StreamApplication("KafkaReadWrite", Graph(computation), appConfig) app } diff --git a/examples/streaming/kafka/src/main/scala/io/gearpump/streaming/examples/kafka/wordcount/KafkaWordCount.scala b/examples/streaming/kafka/src/main/scala/io/gearpump/streaming/examples/kafka/wordcount/KafkaWordCount.scala index 51bc3d65d..6955bcc08 100644 --- a/examples/streaming/kafka/src/main/scala/io/gearpump/streaming/examples/kafka/wordcount/KafkaWordCount.scala +++ b/examples/streaming/kafka/src/main/scala/io/gearpump/streaming/examples/kafka/wordcount/KafkaWordCount.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,31 +19,34 @@ package io.gearpump.streaming.examples.kafka.wordcount import akka.actor.ActorSystem -import io.gearpump.streaming.kafka.lib.KafkaSourceConfig -import io.gearpump.streaming.{StreamApplication, Processor} -import io.gearpump.streaming.kafka.{KafkaSink, KafkaStorageFactory, KafkaSource} -import io.gearpump.streaming.sink.DataSinkProcessor -import io.gearpump.streaming.source.DataSourceProcessor +import kafka.api.OffsetRequest +import org.slf4j.Logger + import io.gearpump.cluster.UserConfig import io.gearpump.cluster.client.ClientContext import io.gearpump.cluster.main.{ArgumentsParser, CLIOption, ParseResult} import io.gearpump.partitioner.HashPartitioner +import io.gearpump.streaming.kafka.lib.KafkaSourceConfig +import io.gearpump.streaming.kafka.{KafkaSink, KafkaSource, KafkaStorageFactory} +import io.gearpump.streaming.sink.DataSinkProcessor +import io.gearpump.streaming.source.DataSourceProcessor +import io.gearpump.streaming.{Processor, StreamApplication} import io.gearpump.util.Graph._ import io.gearpump.util.{AkkaApp, Graph, LogUtil} -import kafka.api.OffsetRequest -import org.slf4j.Logger object KafkaWordCount extends AkkaApp with ArgumentsParser { private val LOG: Logger = LogUtil.getLogger(getClass) override val options: Array[(String, CLIOption[Any])] = Array( - "source" -> CLIOption[Int]("", required = false, defaultValue = Some(1)), + "source" -> CLIOption[Int]("", required = false, + defaultValue = Some(1)), "split" -> CLIOption[Int]("", required = false, defaultValue = Some(1)), "sum" -> CLIOption[Int]("", required = false, defaultValue = Some(1)), - "sink" -> CLIOption[Int]("", required = false, defaultValue = Some(1)) - ) + "sink" -> CLIOption[Int]("", required = false, + defaultValue = Some(1)) + ) - def application(config: ParseResult, system: ActorSystem) : StreamApplication = { + def application(config: ParseResult, system: ActorSystem): StreamApplication = { implicit val actorSystem = system val sourceNum = config.getInt("source") val splitNum = config.getInt("split") @@ -61,7 +64,8 @@ object KafkaWordCount extends AkkaApp with ArgumentsParser { val sink = new KafkaSink("topic2", "localhost:9092") val sinkProcessor = DataSinkProcessor(sink, sinkNum) val partitioner = new HashPartitioner - val computation = sourceProcessor ~ partitioner ~> split ~ partitioner ~> sum ~ partitioner ~> sinkProcessor + val computation = sourceProcessor ~ partitioner ~> split ~ partitioner ~> + sum ~ partitioner ~> sinkProcessor val app = StreamApplication("KafkaWordCount", Graph(computation), appConfig) app } diff --git a/examples/streaming/kafka/src/main/scala/io/gearpump/streaming/examples/kafka/wordcount/Split.scala b/examples/streaming/kafka/src/main/scala/io/gearpump/streaming/examples/kafka/wordcount/Split.scala index 499f1ac51..b46d170f4 100644 --- a/examples/streaming/kafka/src/main/scala/io/gearpump/streaming/examples/kafka/wordcount/Split.scala +++ b/examples/streaming/kafka/src/main/scala/io/gearpump/streaming/examples/kafka/wordcount/Split.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,18 +19,20 @@ package io.gearpump.streaming.examples.kafka.wordcount import com.twitter.bijection.Injection -import io.gearpump.streaming.task.{StartTime, Task, TaskContext} + import io.gearpump.Message import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.task.{StartTime, Task, TaskContext} -class Split(taskContext : TaskContext, conf: UserConfig) extends Task(taskContext, conf) { +class Split(taskContext: TaskContext, conf: UserConfig) extends Task(taskContext, conf) { import taskContext.output - override def onStart(startTime : StartTime) : Unit = { + override def onStart(startTime: StartTime): Unit = { } - override def onNext(msg : Message) : Unit = { - Injection.invert[String, Array[Byte]](msg.msg.asInstanceOf[Array[Byte]]).foreach(_.split("\\s+").foreach( - word => output(new Message(word, msg.timestamp)))) + override def onNext(msg: Message): Unit = { + Injection.invert[String, Array[Byte]](msg.msg.asInstanceOf[Array[Byte]]) + .foreach(_.split("\\s+").foreach( + word => output(new Message(word, msg.timestamp)))) } } diff --git a/examples/streaming/kafka/src/main/scala/io/gearpump/streaming/examples/kafka/wordcount/Sum.scala b/examples/streaming/kafka/src/main/scala/io/gearpump/streaming/examples/kafka/wordcount/Sum.scala index 465754ed0..9c6773362 100644 --- a/examples/streaming/kafka/src/main/scala/io/gearpump/streaming/examples/kafka/wordcount/Sum.scala +++ b/examples/streaming/kafka/src/main/scala/io/gearpump/streaming/examples/kafka/wordcount/Sum.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,18 +19,19 @@ package io.gearpump.streaming.examples.kafka.wordcount import com.twitter.bijection.Injection -import io.gearpump.streaming.task.{StartTime, Task, TaskContext} + import io.gearpump.Message import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.task.{StartTime, Task, TaskContext} -class Sum(taskContext : TaskContext, conf: UserConfig) extends Task(taskContext, conf) { +class Sum(taskContext: TaskContext, conf: UserConfig) extends Task(taskContext, conf) { import taskContext.output private[wordcount] var wordcount = Map.empty[String, Long] - override def onStart(startTime : StartTime) : Unit = {} + override def onStart(startTime: StartTime): Unit = {} - override def onNext(message : Message) : Unit = { + override def onNext(message: Message): Unit = { val word = message.msg.asInstanceOf[String] val count = wordcount.getOrElse(word, 0L) + 1 wordcount += word -> count diff --git a/examples/streaming/kafka/src/test/scala/io/gearpump/streaming/examples/kafka/wordcount/KafkaWordCountSpec.scala b/examples/streaming/kafka/src/test/scala/io/gearpump/streaming/examples/kafka/wordcount/KafkaWordCountSpec.scala index 9e79d7e4d..35f7a6263 100644 --- a/examples/streaming/kafka/src/test/scala/io/gearpump/streaming/examples/kafka/wordcount/KafkaWordCountSpec.scala +++ b/examples/streaming/kafka/src/test/scala/io/gearpump/streaming/examples/kafka/wordcount/KafkaWordCountSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,17 +18,19 @@ package io.gearpump.streaming.examples.kafka.wordcount -import io.gearpump.cluster.ClientToMaster.SubmitApplication -import io.gearpump.cluster.MasterToClient.SubmitApplicationResult -import io.gearpump.cluster.{MasterHarness, TestUtil} -import io.gearpump.util.Util +import scala.concurrent.Future +import scala.util.Success + +import com.typesafe.config.Config import org.scalatest.prop.PropertyChecks import org.scalatest.{BeforeAndAfter, Matchers, PropSpec} -import scala.util.Success -import scala.concurrent.Future +import io.gearpump.cluster.ClientToMaster.SubmitApplication +import io.gearpump.cluster.MasterToClient.SubmitApplicationResult +import io.gearpump.cluster.{MasterHarness, TestUtil} -class KafkaWordCountSpec extends PropSpec with PropertyChecks with Matchers with BeforeAndAfter with MasterHarness { +class KafkaWordCountSpec + extends PropSpec with PropertyChecks with Matchers with BeforeAndAfter with MasterHarness { before { startActorSystem() @@ -38,7 +40,7 @@ class KafkaWordCountSpec extends PropSpec with PropertyChecks with Matchers with shutdownActorSystem() } - override def config = TestUtil.DEFAULT_CONFIG + override def config: Config = TestUtil.DEFAULT_CONFIG property("KafkaWordCount should succeed to submit application with required arguments") { val requiredArgs = Array.empty[String] @@ -58,7 +60,9 @@ class KafkaWordCountSpec extends PropSpec with PropertyChecks with Matchers with forAll(args) { (requiredArgs: Array[String], optionalArgs: Array[String]) => val args = requiredArgs ++ optionalArgs - Future {KafkaWordCount.main(masterConfig, args)} + Future { + KafkaWordCount.main(masterConfig, args) + } masterReceiver.expectMsgType[SubmitApplication](PROCESS_BOOT_TIME) masterReceiver.reply(SubmitApplicationResult(Success(0))) diff --git a/examples/streaming/kafka/src/test/scala/io/gearpump/streaming/examples/kafka/wordcount/SplitSpec.scala b/examples/streaming/kafka/src/test/scala/io/gearpump/streaming/examples/kafka/wordcount/SplitSpec.scala index 864f97db2..2cc6a16fb 100644 --- a/examples/streaming/kafka/src/test/scala/io/gearpump/streaming/examples/kafka/wordcount/SplitSpec.scala +++ b/examples/streaming/kafka/src/test/scala/io/gearpump/streaming/examples/kafka/wordcount/SplitSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,15 +18,14 @@ package io.gearpump.streaming.examples.kafka.wordcount import com.twitter.bijection.Injection -import io.gearpump.streaming.task.TaskContext -import io.gearpump.Message -import io.gearpump.cluster.UserConfig import org.mockito.Matchers._ import org.mockito.Mockito._ import org.scalatest._ import org.scalatest.mock.MockitoSugar -import scala.language.postfixOps +import io.gearpump.Message +import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.task.TaskContext class SplitSpec extends FlatSpec with Matchers with MockitoSugar { diff --git a/examples/streaming/kafka/src/test/scala/io/gearpump/streaming/examples/kafka/wordcount/SumSpec.scala b/examples/streaming/kafka/src/test/scala/io/gearpump/streaming/examples/kafka/wordcount/SumSpec.scala index 6014d0f8c..4dcb9d77f 100644 --- a/examples/streaming/kafka/src/test/scala/io/gearpump/streaming/examples/kafka/wordcount/SumSpec.scala +++ b/examples/streaming/kafka/src/test/scala/io/gearpump/streaming/examples/kafka/wordcount/SumSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,23 +17,24 @@ */ package io.gearpump.streaming.examples.kafka.wordcount -import io.gearpump.streaming.MockUtil -import io.gearpump.streaming.task.StartTime -import io.gearpump.Message -import io.gearpump.cluster.UserConfig +import scala.collection.mutable + import org.mockito.Matchers._ import org.mockito.Mockito._ import org.scalacheck.Gen import org.scalatest.{FlatSpec, Matchers} -import scala.collection.mutable +import io.gearpump.Message +import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.MockUtil +import io.gearpump.streaming.task.StartTime class SumSpec extends FlatSpec with Matchers { it should "sum should calculate the frequency of the word correctly" in { val stringGenerator = Gen.alphaStr - val expectedWordCountMap : mutable.HashMap[String, Long] = new mutable.HashMap[String, Long]() + val expectedWordCountMap: mutable.HashMap[String, Long] = new mutable.HashMap[String, Long]() val taskContext = MockUtil.mockTaskContext @@ -42,7 +43,7 @@ class SumSpec extends FlatSpec with Matchers { val str = "once two two three three three" var totalWordCount = 0 - stringGenerator.map {word => + stringGenerator.map { word => totalWordCount += 1 expectedWordCountMap.put(word, expectedWordCountMap.getOrElse(word, 0L) + 1) sum.onNext(Message(word)) diff --git a/examples/streaming/sol/README.md b/examples/streaming/sol/README.md index e772af49c..a8b10b3bd 100644 --- a/examples/streaming/sol/README.md +++ b/examples/streaming/sol/README.md @@ -1,6 +1,6 @@ SOL is a throughput test. It will create multiple layers, and then do random shuffling between these layers. -SOLPRoducer -> SOLProcessor -> SOLProcessor -> ... +SOLProducer -> SOLProcessor -> SOLProcessor -> ... The original code comes from: https://github.com/yahoo/storm-perf-test diff --git a/examples/streaming/sol/src/main/scala/io/gearpump/streaming/examples/sol/SOL.scala b/examples/streaming/sol/src/main/scala/io/gearpump/streaming/examples/sol/SOL.scala index b531250a9..10c190c31 100644 --- a/examples/streaming/sol/src/main/scala/io/gearpump/streaming/examples/sol/SOL.scala +++ b/examples/streaming/sol/src/main/scala/io/gearpump/streaming/examples/sol/SOL.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,25 +18,30 @@ package io.gearpump.streaming.examples.sol -import io.gearpump.streaming.{StreamApplication, Processor} +import org.slf4j.Logger + import io.gearpump.cluster.UserConfig import io.gearpump.cluster.client.ClientContext import io.gearpump.cluster.main.{ArgumentsParser, CLIOption, ParseResult} import io.gearpump.partitioner.ShufflePartitioner +import io.gearpump.streaming.{Processor, StreamApplication} import io.gearpump.util.Graph._ import io.gearpump.util.{AkkaApp, Graph, LogUtil} -import org.slf4j.Logger object SOL extends AkkaApp with ArgumentsParser { private val LOG: Logger = LogUtil.getLogger(getClass) override val options: Array[(String, CLIOption[Any])] = Array( - "streamProducer"-> CLIOption[Int]("", required = false, defaultValue = Some(1)), - "streamProcessor"-> CLIOption[Int]("", required = false, defaultValue = Some(1)), - "bytesPerMessage" -> CLIOption[Int]("", required = false, defaultValue = Some(100)), - "stages"-> CLIOption[Int]("", required = false, defaultValue = Some(2))) + "streamProducer" -> CLIOption[Int]("", required = false, + defaultValue = Some(1)), + "streamProcessor" -> CLIOption[Int]("", required = false, + defaultValue = Some(1)), + "bytesPerMessage" -> CLIOption[Int]("", required = false, + defaultValue = Some(100)), + "stages" -> CLIOption[Int]("", required = false, + defaultValue = Some(2))) - def application(config: ParseResult) : StreamApplication = { + def application(config: ParseResult): StreamApplication = { val spoutNum = config.getInt("streamProducer") val boltNum = config.getInt("streamProcessor") val bytesPerMessage = config.getInt("bytesPerMessage") diff --git a/examples/streaming/sol/src/main/scala/io/gearpump/streaming/examples/sol/SOLStreamProcessor.scala b/examples/streaming/sol/src/main/scala/io/gearpump/streaming/examples/sol/SOLStreamProcessor.scala index 22c36ef83..de1054f9f 100644 --- a/examples/streaming/sol/src/main/scala/io/gearpump/streaming/examples/sol/SOLStreamProcessor.scala +++ b/examples/streaming/sol/src/main/scala/io/gearpump/streaming/examples/sol/SOLStreamProcessor.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,44 +19,47 @@ package io.gearpump.streaming.examples.sol import java.util.concurrent.TimeUnit +import scala.concurrent.duration.FiniteDuration import akka.actor.Cancellable -import io.gearpump.streaming.task.{StartTime, Task, TaskContext} + import io.gearpump.Message import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.task.{StartTime, Task, TaskContext} -import scala.concurrent.duration.FiniteDuration - -class SOLStreamProcessor(taskContext : TaskContext, conf: UserConfig) extends Task(taskContext, conf) { +class SOLStreamProcessor(taskContext: TaskContext, conf: UserConfig) + extends Task(taskContext, conf) { import taskContext.output val taskConf = taskContext - private var msgCount : Long = 0 - private var scheduler : Cancellable = null - private var snapShotWordCount : Long = 0 - private var snapShotTime : Long = 0 + private var msgCount: Long = 0 + private var scheduler: Cancellable = null + private var snapShotWordCount: Long = 0 + private var snapShotTime: Long = 0 - override def onStart(startTime : StartTime) : Unit = { + override def onStart(startTime: StartTime): Unit = { scheduler = taskContext.schedule(new FiniteDuration(5, TimeUnit.SECONDS), new FiniteDuration(5, TimeUnit.SECONDS))(reportWordCount()) snapShotTime = System.currentTimeMillis() } - override def onNext(msg : Message) : Unit = { + override def onNext(msg: Message): Unit = { output(msg) msgCount = msgCount + 1 } - override def onStop() : Unit = { + override def onStop(): Unit = { if (scheduler != null) { scheduler.cancel() } } - def reportWordCount() : Unit = { - val current : Long = System.currentTimeMillis() - LOG.info(s"Task ${taskConf.taskId} Throughput: ${(msgCount - snapShotWordCount, (current - snapShotTime) / 1000)} (words, second)") + def reportWordCount(): Unit = { + val current: Long = System.currentTimeMillis() + LOG.info(s"Task ${taskConf.taskId} " + + s"Throughput: ${(msgCount - snapShotWordCount, (current - snapShotTime) / 1000)} " + + s"(words, second)") snapShotWordCount = msgCount snapShotTime = current } diff --git a/examples/streaming/sol/src/main/scala/io/gearpump/streaming/examples/sol/SOLStreamProducer.scala b/examples/streaming/sol/src/main/scala/io/gearpump/streaming/examples/sol/SOLStreamProducer.scala index 7f285f216..5c0f3be74 100644 --- a/examples/streaming/sol/src/main/scala/io/gearpump/streaming/examples/sol/SOLStreamProducer.scala +++ b/examples/streaming/sol/src/main/scala/io/gearpump/streaming/examples/sol/SOLStreamProducer.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,22 +20,23 @@ package io.gearpump.streaming.examples.sol import java.util.Random -import io.gearpump.streaming.task.{StartTime, Task, TaskContext} import io.gearpump.Message import io.gearpump.cluster.UserConfig import io.gearpump.streaming.examples.sol.SOLStreamProducer._ +import io.gearpump.streaming.task.{StartTime, Task, TaskContext} -class SOLStreamProducer(taskContext : TaskContext, conf: UserConfig) extends Task(taskContext, conf) { +class SOLStreamProducer(taskContext: TaskContext, conf: UserConfig) + extends Task(taskContext, conf) { import taskContext.output private val sizeInBytes = conf.getInt(SOLStreamProducer.BYTES_PER_MESSAGE) - .getOrElse(DEFAULT_MESSAGE_SIZE) - private var messages : Array[String] = null - private var rand : Random = null - private var messageCount : Long = 0 + .getOrElse(DEFAULT_MESSAGE_SIZE) + private var messages: Array[String] = null + private var rand: Random = null + private var messageCount: Long = 0 - override def onStart(startTime : StartTime) : Unit = { + override def onStart(startTime: StartTime): Unit = { prepareRandomMessage self ! Start } @@ -47,16 +48,16 @@ class SOLStreamProducer(taskContext : TaskContext, conf: UserConfig) extends Tas 0.until(differentMessages).map { index => val sb = new StringBuilder(sizeInBytes) - //Even though java encodes strings in UCS2, the serialized version sent by the tuples + // Even though java encodes strings in UCS2, the serialized version sent by the tuples // is UTF8, so it should be a single byte - 0.until(sizeInBytes).foldLeft(sb){(sb, j) => - sb.append(rand.nextInt(9)); + 0.until(sizeInBytes).foldLeft(sb) { (sb, j) => + sb.append(rand.nextInt(9)) } - messages(index) = sb.toString(); + messages(index) = sb.toString() } } - override def onNext(msg : Message) : Unit = { + override def onNext(msg: Message): Unit = { val message = messages(rand.nextInt(messages.length)) output(new Message(message, System.currentTimeMillis())) messageCount = messageCount + 1L @@ -64,13 +65,14 @@ class SOLStreamProducer(taskContext : TaskContext, conf: UserConfig) extends Tas } // messageSourceMinClock represent the min clock of the message source - private def messageSourceMinClock : Message = { + private def messageSourceMinClock: Message = { Message("tick", System.currentTimeMillis()) } } object SOLStreamProducer { - val DEFAULT_MESSAGE_SIZE = 100 // bytes + val DEFAULT_MESSAGE_SIZE = 100 + // Bytes val BYTES_PER_MESSAGE = "bytesPerMessage" val Start = Message("start") } diff --git a/examples/streaming/sol/src/test/scala/io/gearpump/streaming/examples/sol/SOLSpec.scala b/examples/streaming/sol/src/test/scala/io/gearpump/streaming/examples/sol/SOLSpec.scala index 1f3dbbd5e..6e266d032 100644 --- a/examples/streaming/sol/src/test/scala/io/gearpump/streaming/examples/sol/SOLSpec.scala +++ b/examples/streaming/sol/src/test/scala/io/gearpump/streaming/examples/sol/SOLSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,16 +18,19 @@ package io.gearpump.streaming.examples.sol -import io.gearpump.cluster.ClientToMaster.SubmitApplication -import io.gearpump.cluster.MasterToClient.SubmitApplicationResult -import io.gearpump.cluster.{MasterHarness, TestUtil} +import scala.concurrent.Future +import scala.util.Success + +import com.typesafe.config.Config import org.scalatest.prop.PropertyChecks import org.scalatest.{BeforeAndAfterAll, Matchers, PropSpec} -import scala.concurrent.Future -import scala.util.Success +import io.gearpump.cluster.ClientToMaster.SubmitApplication +import io.gearpump.cluster.MasterToClient.SubmitApplicationResult +import io.gearpump.cluster.{MasterHarness, TestUtil} -class SOLSpec extends PropSpec with PropertyChecks with Matchers with BeforeAndAfterAll with MasterHarness { +class SOLSpec + extends PropSpec with PropertyChecks with Matchers with BeforeAndAfterAll with MasterHarness { override def beforeAll { startActorSystem() } @@ -36,7 +39,7 @@ class SOLSpec extends PropSpec with PropertyChecks with Matchers with BeforeAndA shutdownActorSystem() } - override def config = TestUtil.DEFAULT_CONFIG + override def config: Config = TestUtil.DEFAULT_CONFIG property("SOL should succeed to submit application with required arguments") { val requiredArgs = Array.empty[String] @@ -56,11 +59,12 @@ class SOLSpec extends PropSpec with PropertyChecks with Matchers with BeforeAndA forAll(args) { (requiredArgs: Array[String], optionalArgs: Array[String]) => val args = requiredArgs ++ optionalArgs - Future {SOL.main(masterConfig, args)} + Future { + SOL.main(masterConfig, args) + } masterReceiver.expectMsgType[SubmitApplication](PROCESS_BOOT_TIME) masterReceiver.reply(SubmitApplicationResult(Success(0))) } } - } diff --git a/examples/streaming/sol/src/test/scala/io/gearpump/streaming/examples/sol/SOLStreamProcessorSpec.scala b/examples/streaming/sol/src/test/scala/io/gearpump/streaming/examples/sol/SOLStreamProcessorSpec.scala index acc7b8f53..f5035b4c7 100644 --- a/examples/streaming/sol/src/test/scala/io/gearpump/streaming/examples/sol/SOLStreamProcessorSpec.scala +++ b/examples/streaming/sol/src/test/scala/io/gearpump/streaming/examples/sol/SOLStreamProcessorSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,15 +17,14 @@ */ package io.gearpump.streaming.examples.sol -import io.gearpump.streaming.MockUtil -import io.gearpump.streaming.task.StartTime -import io.gearpump.Message -import io.gearpump.cluster.UserConfig import org.mockito.Mockito._ import org.scalacheck.Gen import org.scalatest.{FlatSpec, Matchers} -import scala.language.postfixOps +import io.gearpump.Message +import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.MockUtil +import io.gearpump.streaming.task.StartTime class SOLStreamProcessorSpec extends FlatSpec with Matchers { diff --git a/examples/streaming/sol/src/test/scala/io/gearpump/streaming/examples/sol/SOLStreamProducerSpec.scala b/examples/streaming/sol/src/test/scala/io/gearpump/streaming/examples/sol/SOLStreamProducerSpec.scala index 4ff7d126d..4bac30cf3 100644 --- a/examples/streaming/sol/src/test/scala/io/gearpump/streaming/examples/sol/SOLStreamProducerSpec.scala +++ b/examples/streaming/sol/src/test/scala/io/gearpump/streaming/examples/sol/SOLStreamProducerSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,14 +17,15 @@ */ package io.gearpump.streaming.examples.sol -import io.gearpump.streaming.MockUtil -import io.gearpump.streaming.task.StartTime -import io.gearpump.Message -import io.gearpump.cluster.UserConfig import org.mockito.Matchers._ import org.mockito.Mockito._ import org.scalatest.{Matchers, WordSpec} +import io.gearpump.Message +import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.MockUtil +import io.gearpump.streaming.task.StartTime + class SOLStreamProducerSpec extends WordSpec with Matchers { "SOLStreamProducer" should { diff --git a/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/MessageCountApp.scala b/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/MessageCountApp.scala index e9ceb8b8c..3c6cde658 100644 --- a/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/MessageCountApp.scala +++ b/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/MessageCountApp.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,22 +19,24 @@ package io.gearpump.streaming.examples.state import akka.actor.ActorSystem -import io.gearpump.streaming.kafka.{KafkaSink, KafkaSource, KafkaStorageFactory} -import io.gearpump.streaming.sink.DataSinkProcessor -import io.gearpump.streaming.source.DataSourceProcessor -import io.gearpump.streaming.{StreamApplication, Processor} -import io.gearpump.streaming.examples.state.processor.CountProcessor -import io.gearpump.streaming.hadoop.HadoopCheckpointStoreFactory -import io.gearpump.streaming.hadoop.lib.rotation.FileSizeRotation -import io.gearpump.streaming.state.impl.PersistentStateConfig +import org.apache.hadoop.conf.Configuration + import io.gearpump.cluster.UserConfig import io.gearpump.cluster.client.ClientContext import io.gearpump.cluster.main.{ArgumentsParser, CLIOption, ParseResult} import io.gearpump.partitioner.HashPartitioner +import io.gearpump.streaming.examples.state.processor.CountProcessor +import io.gearpump.streaming.hadoop.HadoopCheckpointStoreFactory +import io.gearpump.streaming.hadoop.lib.rotation.FileSizeRotation +import io.gearpump.streaming.kafka.{KafkaSink, KafkaSource, KafkaStorageFactory} +import io.gearpump.streaming.sink.DataSinkProcessor +import io.gearpump.streaming.source.DataSourceProcessor +import io.gearpump.streaming.state.impl.PersistentStateConfig +import io.gearpump.streaming.{Processor, StreamApplication} import io.gearpump.util.Graph.Node import io.gearpump.util.{AkkaApp, Graph} -import org.apache.hadoop.conf.Configuration +/** Does exactly-once message count */ object MessageCountApp extends AkkaApp with ArgumentsParser { val SOURCE_TASK = "sourceTask" val COUNT_TASK = "countTask" @@ -46,21 +48,25 @@ object MessageCountApp extends AkkaApp with ArgumentsParser { val DEFAULT_FS = "defaultFS" override val options: Array[(String, CLIOption[Any])] = Array( - SOURCE_TASK -> CLIOption[Int]("", required = false, defaultValue = Some(1)), + SOURCE_TASK -> CLIOption[Int]("", required = false, + defaultValue = Some(1)), COUNT_TASK -> CLIOption("", required = false, defaultValue = Some(1)), - SINK_TASK -> CLIOption[Int]("", required = false, defaultValue = Some(1)), + SINK_TASK -> CLIOption[Int]("", required = false, + defaultValue = Some(1)), SOURCE_TOPIC -> CLIOption[String]("", required = true), SINK_TOPIC -> CLIOption[String]("", required = true), - ZOOKEEPER_CONNECT -> CLIOption[String]("", required = true), + ZOOKEEPER_CONNECT -> CLIOption[String]("", + required = true), BROKER_LIST -> CLIOption[String]("", required = true), - DEFAULT_FS -> CLIOption[String]("", required = true) + DEFAULT_FS -> CLIOption[String]("", + required = true) ) - def application(config: ParseResult)(implicit system: ActorSystem) : StreamApplication = { + def application(config: ParseResult)(implicit system: ActorSystem): StreamApplication = { val hadoopConfig = new Configuration hadoopConfig.set("fs.defaultFS", config.getString(DEFAULT_FS)) val checkpointStoreFactory = new HadoopCheckpointStoreFactory("MessageCount", hadoopConfig, - // rotate on 1KB + // Rotates on 1KB new FileSizeRotation(1000)) val taskConfig = UserConfig.empty .withBoolean(PersistentStateConfig.STATE_CHECKPOINT_ENABLE, true) @@ -77,7 +83,8 @@ object MessageCountApp extends AkkaApp with ArgumentsParser { val kafkaSink = new KafkaSink(config.getString(SINK_TOPIC), brokerList) val sinkProcessor = DataSinkProcessor(kafkaSink, config.getInt(SINK_TASK)) val partitioner = new HashPartitioner() - val graph = Graph(sourceProcessor ~ partitioner ~> countProcessor ~ partitioner ~> sinkProcessor) + val graph = Graph(sourceProcessor ~ partitioner + ~> countProcessor ~ partitioner ~> sinkProcessor) val app = StreamApplication("MessageCount", graph, UserConfig.empty) app } diff --git a/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/WindowAverageApp.scala b/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/WindowAverageApp.scala index 748980856..6f3bc7954 100644 --- a/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/WindowAverageApp.scala +++ b/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/WindowAverageApp.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,40 +19,45 @@ package io.gearpump.streaming.examples.state import akka.actor.ActorSystem -import io.gearpump.streaming.{StreamApplication, Processor} -import io.gearpump.streaming.examples.state.processor.{WindowAverageProcessor, NumberGeneratorProcessor} -import io.gearpump.streaming.hadoop.HadoopCheckpointStoreFactory -import io.gearpump.streaming.state.impl.{WindowConfig, PersistentStateConfig} +import org.apache.hadoop.conf.Configuration + import io.gearpump.cluster.UserConfig import io.gearpump.cluster.client.ClientContext import io.gearpump.cluster.main.{ArgumentsParser, CLIOption, ParseResult} import io.gearpump.partitioner.HashPartitioner +import io.gearpump.streaming.examples.state.processor.{NumberGeneratorProcessor, WindowAverageProcessor} +import io.gearpump.streaming.hadoop.HadoopCheckpointStoreFactory +import io.gearpump.streaming.state.impl.{PersistentStateConfig, WindowConfig} +import io.gearpump.streaming.{Processor, StreamApplication} import io.gearpump.util.Graph.Node import io.gearpump.util.{AkkaApp, Graph} -import org.apache.hadoop.conf.Configuration +/** Does exactly-once sliding window based average aggregation */ object WindowAverageApp extends AkkaApp with ArgumentsParser { override val options: Array[(String, CLIOption[Any])] = Array( "gen" -> CLIOption("", required = false, defaultValue = Some(1)), "window" -> CLIOption(" CLIOption("", required = false , defaultValue = Some(5000)), - "window_step" -> CLIOption("", required = false , defaultValue = Some(5000)) + "window_size" -> CLIOption("", required = false, + defaultValue = Some(5000)), + "window_step" -> CLIOption("", required = false, + defaultValue = Some(5000)) ) - def application(config: ParseResult)(implicit system: ActorSystem) : StreamApplication = { + def application(config: ParseResult)(implicit system: ActorSystem): StreamApplication = { val windowSize = config.getInt("window_size") val windowStep = config.getInt("window_step") val checkpointStoreFactory = new HadoopCheckpointStoreFactory("MessageCount", new Configuration) - val taskConfig = UserConfig.empty - .withBoolean(PersistentStateConfig.STATE_CHECKPOINT_ENABLE, true) - .withLong(PersistentStateConfig.STATE_CHECKPOINT_INTERVAL_MS, 1000L) - .withValue(PersistentStateConfig.STATE_CHECKPOINT_STORE_FACTORY, checkpointStoreFactory) - .withValue(WindowConfig.NAME, WindowConfig(windowSize, windowStep)) + val taskConfig = UserConfig.empty. + withBoolean(PersistentStateConfig.STATE_CHECKPOINT_ENABLE, true) + .withLong(PersistentStateConfig.STATE_CHECKPOINT_INTERVAL_MS, 1000L) + .withValue(PersistentStateConfig.STATE_CHECKPOINT_STORE_FACTORY, checkpointStoreFactory) + .withValue(WindowConfig.NAME, WindowConfig(windowSize, windowStep)) val gen = Processor[NumberGeneratorProcessor](config.getInt("gen")) val count = Processor[WindowAverageProcessor](config.getInt("window"), taskConf = taskConfig) val partitioner = new HashPartitioner() - val app = StreamApplication("WindowAverage", Graph(gen ~ partitioner ~> count), UserConfig.empty) + val app = StreamApplication("WindowAverage", Graph(gen ~ partitioner ~> count), + UserConfig.empty) app } diff --git a/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/processor/CountProcessor.scala b/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/processor/CountProcessor.scala index 04cea6904..6610b91da 100644 --- a/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/processor/CountProcessor.scala +++ b/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/processor/CountProcessor.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,13 +18,13 @@ package io.gearpump.streaming.examples.state.processor +import io.gearpump.Message +import io.gearpump.cluster.UserConfig import io.gearpump.streaming.monoid.AlgebirdMonoid import io.gearpump.streaming.serializer.ChillSerializer -import io.gearpump.streaming.state.api.{PersistentTask, PersistentState} +import io.gearpump.streaming.state.api.{PersistentState, PersistentTask} import io.gearpump.streaming.state.impl.NonWindowState import io.gearpump.streaming.task.TaskContext -import io.gearpump.cluster.UserConfig -import io.gearpump.Message class CountProcessor(taskContext: TaskContext, conf: UserConfig) extends PersistentTask[Int](taskContext, conf) { @@ -39,4 +39,3 @@ class CountProcessor(taskContext: TaskContext, conf: UserConfig) } } - diff --git a/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/processor/NumberGeneratorProcessor.scala b/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/processor/NumberGeneratorProcessor.scala index 0dae825f5..fa9854f2a 100644 --- a/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/processor/NumberGeneratorProcessor.scala +++ b/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/processor/NumberGeneratorProcessor.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,11 +18,12 @@ package io.gearpump.streaming.examples.state.processor -import io.gearpump.streaming.task.{StartTime, Task, TaskContext} import io.gearpump.Message import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.task.{StartTime, Task, TaskContext} -class NumberGeneratorProcessor(taskContext : TaskContext, conf: UserConfig) extends Task(taskContext, conf) { +class NumberGeneratorProcessor(taskContext: TaskContext, conf: UserConfig) + extends Task(taskContext, conf) { import taskContext.output private var num = 0L diff --git a/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/processor/WindowAverageProcessor.scala b/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/processor/WindowAverageProcessor.scala index 82d624394..e7ac9b3f1 100644 --- a/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/processor/WindowAverageProcessor.scala +++ b/examples/streaming/state/src/main/scala/io/gearpump/streaming/examples/state/processor/WindowAverageProcessor.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,24 +18,25 @@ package io.gearpump.streaming.examples.state.processor +import scala.collection.immutable.TreeMap + import com.twitter.algebird.{AveragedGroup, AveragedValue} +import org.slf4j.Logger + +import io.gearpump.Message +import io.gearpump.cluster.UserConfig import io.gearpump.streaming.monoid.AlgebirdGroup import io.gearpump.streaming.serializer.ChillSerializer -import io.gearpump.streaming.state.api.{PersistentTask, PersistentState} -import io.gearpump.streaming.state.impl.{WindowConfig, WindowState, Interval, Window} +import io.gearpump.streaming.state.api.{PersistentState, PersistentTask} +import io.gearpump.streaming.state.impl.{Interval, Window, WindowConfig, WindowState} import io.gearpump.streaming.task.TaskContext -import io.gearpump.Message -import io.gearpump.cluster.UserConfig import io.gearpump.util.LogUtil -import org.slf4j.Logger - -import scala.collection.immutable.TreeMap object WindowAverageProcessor { val LOG: Logger = LogUtil.getLogger(classOf[WindowAverageProcessor]) } -class WindowAverageProcessor(taskContext : TaskContext, conf: UserConfig) +class WindowAverageProcessor(taskContext: TaskContext, conf: UserConfig) extends PersistentTask[AveragedValue](taskContext, conf) { override def persistentState: PersistentState[AveragedValue] = { @@ -46,7 +47,7 @@ class WindowAverageProcessor(taskContext : TaskContext, conf: UserConfig) } override def processMessage(state: PersistentState[AveragedValue], - message: Message): Unit = { + message: Message): Unit = { val value = AveragedValue(message.msg.asInstanceOf[String].toLong) state.update(message.timestamp, value) } diff --git a/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/MessageCountAppSpec.scala b/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/MessageCountAppSpec.scala index 8053f57b7..040c3434a 100644 --- a/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/MessageCountAppSpec.scala +++ b/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/MessageCountAppSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,18 +18,19 @@ package io.gearpump.streaming.examples.state -import io.gearpump.streaming.examples.state.MessageCountApp._ +import scala.concurrent.Future +import scala.util.Success + +import org.scalatest.prop.PropertyChecks +import org.scalatest.{BeforeAndAfter, Matchers, PropSpec} + import io.gearpump.cluster.ClientToMaster.SubmitApplication import io.gearpump.cluster.MasterToClient.SubmitApplicationResult import io.gearpump.cluster.{MasterHarness, TestUtil} -import org.scalatest.{Matchers, BeforeAndAfter, PropSpec} -import org.scalatest.prop.PropertyChecks - - -import scala.concurrent.Future -import scala.util.Success +import io.gearpump.streaming.examples.state.MessageCountApp._ -class MessageCountAppSpec extends PropSpec with PropertyChecks with Matchers with BeforeAndAfter with MasterHarness { +class MessageCountAppSpec + extends PropSpec with PropertyChecks with Matchers with BeforeAndAfter with MasterHarness { before { startActorSystem() @@ -39,7 +40,7 @@ class MessageCountAppSpec extends PropSpec with PropertyChecks with Matchers wit shutdownActorSystem() } - override def config = TestUtil.DEFAULT_CONFIG + protected override def config = TestUtil.DEFAULT_CONFIG property("MessageCount should succeed to submit application with required arguments") { val requiredArgs = Array( @@ -68,7 +69,9 @@ class MessageCountAppSpec extends PropSpec with PropertyChecks with Matchers wit val masterReceiver = createMockMaster() forAll(args) { (requiredArgs: Array[String], optionalArgs: Array[String]) => val args = requiredArgs ++ optionalArgs - Future {MessageCountApp.main(masterConfig, args)} + Future { + MessageCountApp.main(masterConfig, args) + } masterReceiver.expectMsgType[SubmitApplication](PROCESS_BOOT_TIME) masterReceiver.reply(SubmitApplicationResult(Success(0))) } diff --git a/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/WindowAverageAppSpec.scala b/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/WindowAverageAppSpec.scala index 998dd1d1d..7c1c79890 100644 --- a/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/WindowAverageAppSpec.scala +++ b/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/WindowAverageAppSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,16 +18,20 @@ package io.gearpump.streaming.examples.state +import scala.concurrent.Future +import scala.util.Success + +import com.typesafe.config.Config +import org.scalatest.prop.PropertyChecks +import org.scalatest.{BeforeAndAfter, Matchers, PropSpec} + import io.gearpump.cluster.ClientToMaster.SubmitApplication import io.gearpump.cluster.MasterToClient.SubmitApplicationResult import io.gearpump.cluster.{MasterHarness, TestUtil} -import org.scalatest.{Matchers, BeforeAndAfter, PropSpec} -import org.scalatest.prop.PropertyChecks -import scala.concurrent.Future -import scala.util.Success +class WindowAverageAppSpec + extends PropSpec with PropertyChecks with Matchers with BeforeAndAfter with MasterHarness { -class WindowAverageAppSpec extends PropSpec with PropertyChecks with Matchers with BeforeAndAfter with MasterHarness { before { startActorSystem() } @@ -36,7 +40,7 @@ class WindowAverageAppSpec extends PropSpec with PropertyChecks with Matchers wi shutdownActorSystem() } - override def config = TestUtil.DEFAULT_CONFIG + override def config: Config = TestUtil.DEFAULT_CONFIG property("WindowAverage should succeed to submit application with required arguments") { val requiredArgs = Array.empty[String] @@ -61,7 +65,9 @@ class WindowAverageAppSpec extends PropSpec with PropertyChecks with Matchers wi forAll(args) { (requiredArgs: Array[String], optionalArgs: Array[String]) => val args = requiredArgs ++ optionalArgs - Future {WindowAverageApp.main(masterConfig, args)} + Future { + WindowAverageApp.main(masterConfig, args) + } masterReceiver.expectMsgType[SubmitApplication](PROCESS_BOOT_TIME) masterReceiver.reply(SubmitApplicationResult(Success(0))) diff --git a/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/processor/CountProcessorSpec.scala b/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/processor/CountProcessorSpec.scala index cbc9cce45..a69116b95 100644 --- a/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/processor/CountProcessorSpec.scala +++ b/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/processor/CountProcessorSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,23 +18,23 @@ package io.gearpump.streaming.examples.state.processor +import scala.concurrent.Await +import scala.concurrent.duration._ + import akka.actor.ActorSystem import akka.testkit.TestProbe -import io.gearpump.streaming.MockUtil -import io.gearpump.streaming.state.api.PersistentTask -import io.gearpump.streaming.state.impl.PersistentStateConfig -import io.gearpump.streaming.task.ReportCheckpointClock -import io.gearpump.streaming.transaction.api.CheckpointStoreFactory -import io.gearpump.Message -import io.gearpump.cluster.UserConfig -import io.gearpump.streaming.state.impl.InMemoryCheckpointStoreFactory -import io.gearpump.streaming.task.StartTime import org.mockito.Mockito._ import org.scalacheck.Gen -import org.scalatest.{Matchers, PropSpec} import org.scalatest.prop.PropertyChecks +import org.scalatest.{Matchers, PropSpec} -import scala.concurrent.duration._ +import io.gearpump.Message +import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.MockUtil +import io.gearpump.streaming.state.api.PersistentTask +import io.gearpump.streaming.state.impl.{InMemoryCheckpointStoreFactory, PersistentStateConfig} +import io.gearpump.streaming.task.{ReportCheckpointClock, StartTime} +import io.gearpump.streaming.transaction.api.CheckpointStoreFactory class CountProcessorSpec extends PropSpec with PropertyChecks with Matchers { @@ -51,7 +51,8 @@ class CountProcessorSpec extends PropSpec with PropertyChecks with Matchers { val conf = UserConfig.empty .withBoolean(PersistentStateConfig.STATE_CHECKPOINT_ENABLE, true) .withLong(PersistentStateConfig.STATE_CHECKPOINT_INTERVAL_MS, num) - .withValue[CheckpointStoreFactory](PersistentStateConfig.STATE_CHECKPOINT_STORE_FACTORY, new InMemoryCheckpointStoreFactory) + .withValue[CheckpointStoreFactory](PersistentStateConfig.STATE_CHECKPOINT_STORE_FACTORY, + new InMemoryCheckpointStoreFactory) val count = new CountProcessor(taskContext, conf) @@ -61,25 +62,23 @@ class CountProcessorSpec extends PropSpec with PropertyChecks with Matchers { count.onStart(StartTime(0L)) appMaster.expectMsg(ReportCheckpointClock(taskContext.taskId, 0L)) - for (i <- 0L to num) { count.onNext(Message("", i)) count.state.get shouldBe Some(i + 1) } - // next checkpoint time is at num - // not yet + // Next checkpoint time is not arrived yet when(taskContext.upstreamMinClock).thenReturn(0L) count.onNext(PersistentTask.CHECKPOINT) - appMaster.expectNoMsg(10 milliseconds) + appMaster.expectNoMsg(10.milliseconds) - // time to checkpoint + // Time to checkpoint when(taskContext.upstreamMinClock).thenReturn(num) count.onNext(PersistentTask.CHECKPOINT) - // only the state before checkpoint time is checkpointed + // Only the state before checkpoint time is checkpointed appMaster.expectMsg(ReportCheckpointClock(taskContext.taskId, num)) } - system.shutdown() - system.awaitTermination() + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } } diff --git a/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/processor/NumberGeneratorProcessorSpec.scala b/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/processor/NumberGeneratorProcessorSpec.scala index 8105c32fd..18a49ac2e 100644 --- a/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/processor/NumberGeneratorProcessorSpec.scala +++ b/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/processor/NumberGeneratorProcessorSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,16 +18,20 @@ package io.gearpump.streaming.examples.state.processor +import scala.concurrent.Await +import scala.concurrent.duration.Duration + import akka.actor.ActorSystem import akka.testkit.TestProbe -import io.gearpump.streaming.MockUtil -import io.gearpump.streaming.task.StartTime -import io.gearpump.Message -import io.gearpump.cluster.UserConfig -import org.mockito.{Matchers => MockitoMatchers} import org.mockito.Mockito._ +import org.mockito.{Matchers => MockitoMatchers} import org.scalatest.{Matchers, WordSpec} +import io.gearpump.Message +import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.MockUtil +import io.gearpump.streaming.task.StartTime + class NumberGeneratorProcessorSpec extends WordSpec with Matchers { "NumberGeneratorProcessor" should { "send random numbers" in { @@ -38,7 +42,7 @@ class NumberGeneratorProcessorSpec extends WordSpec with Matchers { val mockTaskActor = TestProbe() - //mock self ActorRef + // Mock self ActorRef when(taskContext.self).thenReturn(mockTaskActor.ref) val conf = UserConfig.empty @@ -46,13 +50,12 @@ class NumberGeneratorProcessorSpec extends WordSpec with Matchers { genNum.onStart(StartTime(0)) mockTaskActor.expectMsgType[Message] - genNum.onNext(Message("next")) verify(taskContext).output(MockitoMatchers.any[Message]) - //mockTaskActor.expectMsgType[Message] + // mockTaskActor.expectMsgType[Message] - system.shutdown() - system.awaitTermination() + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } } } diff --git a/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/processor/WindowAverageProcessorSpec.scala b/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/processor/WindowAverageProcessorSpec.scala index 03bc8fb3b..a9c52aac2 100644 --- a/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/processor/WindowAverageProcessorSpec.scala +++ b/examples/streaming/state/src/test/scala/io/gearpump/streaming/examples/state/processor/WindowAverageProcessorSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,25 +18,24 @@ package io.gearpump.streaming.examples.state.processor +import scala.concurrent.Await +import scala.concurrent.duration._ + import akka.actor.ActorSystem import akka.testkit.TestProbe import com.twitter.algebird.AveragedValue -import io.gearpump.streaming.MockUtil -import io.gearpump.streaming.state.api.PersistentTask -import io.gearpump.streaming.state.impl.{WindowConfig, PersistentStateConfig} -import io.gearpump.streaming.task.ReportCheckpointClock -import io.gearpump.streaming.transaction.api.CheckpointStoreFactory -import io.gearpump.Message -import io.gearpump.cluster.UserConfig -import io.gearpump.streaming.state.impl.InMemoryCheckpointStoreFactory -import io.gearpump.streaming.task.StartTime import org.mockito.Mockito._ import org.scalacheck.Gen -import org.scalatest.{Matchers, PropSpec} import org.scalatest.prop.PropertyChecks +import org.scalatest.{Matchers, PropSpec} -import scala.concurrent.duration._ - +import io.gearpump.Message +import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.MockUtil +import io.gearpump.streaming.state.api.PersistentTask +import io.gearpump.streaming.state.impl.{InMemoryCheckpointStoreFactory, PersistentStateConfig, WindowConfig} +import io.gearpump.streaming.task.{ReportCheckpointClock, StartTime} +import io.gearpump.streaming.transaction.api.CheckpointStoreFactory class WindowAverageProcessorSpec extends PropSpec with PropertyChecks with Matchers { property("WindowAverageProcessor should update state") { @@ -51,10 +50,11 @@ class WindowAverageProcessorSpec extends PropSpec with PropertyChecks with Match val windowStep = num val conf = UserConfig.empty - .withBoolean(PersistentStateConfig.STATE_CHECKPOINT_ENABLE, true) - .withLong(PersistentStateConfig.STATE_CHECKPOINT_INTERVAL_MS, num) - .withValue[CheckpointStoreFactory](PersistentStateConfig.STATE_CHECKPOINT_STORE_FACTORY, new InMemoryCheckpointStoreFactory) - .withValue(WindowConfig.NAME, WindowConfig(windowSize, windowStep)) + .withBoolean(PersistentStateConfig.STATE_CHECKPOINT_ENABLE, true) + .withLong(PersistentStateConfig.STATE_CHECKPOINT_INTERVAL_MS, num) + .withValue[CheckpointStoreFactory](PersistentStateConfig.STATE_CHECKPOINT_STORE_FACTORY, + new InMemoryCheckpointStoreFactory) + .withValue(WindowConfig.NAME, WindowConfig(windowSize, windowStep)) val windowAverage = new WindowAverageProcessor(taskContext, conf) @@ -69,19 +69,18 @@ class WindowAverageProcessorSpec extends PropSpec with PropertyChecks with Match windowAverage.state.get shouldBe Some(AveragedValue(i + 1, data)) } - // next checkpoint time is at num - // not yet + // Next checkpoint time is not arrived yet when(taskContext.upstreamMinClock).thenReturn(0L) windowAverage.onNext(PersistentTask.CHECKPOINT) - appMaster.expectNoMsg(10 milliseconds) + appMaster.expectNoMsg(10.milliseconds) - // time to checkpoint + // Time to checkpoint when(taskContext.upstreamMinClock).thenReturn(num) windowAverage.onNext(PersistentTask.CHECKPOINT) appMaster.expectMsg(ReportCheckpointClock(taskContext.taskId, num)) } - system.shutdown() - system.awaitTermination() + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } } diff --git a/examples/streaming/stockcrawler/src/main/resources/stock/css/custom.css b/examples/streaming/stockcrawler/src/main/resources/stock/css/custom.css index 4693dc680..182d7228c 100644 --- a/examples/streaming/stockcrawler/src/main/resources/stock/css/custom.css +++ b/examples/streaming/stockcrawler/src/main/resources/stock/css/custom.css @@ -1,16 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + .ui-datepicker { font-size: 11px; - } +} + .sidebar-label { - font-size:15px; + font-size: 15px; font-family: calibri, Arial, Helvetica, sans-serif; } - + .help { - font-size:12px; + font-size: 12px; font-family: calibri, Arial, Helvetica, sans-serif; } - + div.splitter { margin: 12px 0px 7px 0px; clear: both; @@ -18,13 +37,13 @@ div.splitter { } input.sidebar { - width:165px + width: 165px } select.sidebar { - width:198px + width: 198px } - + table.dataintable { font-family: calibri, Arial, Helvetica, sans-serif; font-size: 15px; @@ -48,49 +67,49 @@ table.dataintable td { border: 1px solid #AAA; } -#search{ - width:100px; - height:25px; - position:relative; - left:0px; - top:5px; +#search { + width: 100px; + height: 25px; + position: relative; + left: 0px; + top: 5px; } -#mytable{ - width:100%; - height:300; - float:left; +#mytable { + width: 100%; + height: 300; + float: left; } -#mychart{ - height:250px; - width:100%; +#mychart { + height: 250px; + width: 100%; } -#Menu{ - height:100%; - width:245px; - float:left; +#Menu { + height: 100%; + width: 245px; + float: left; } -#header{ - height:115px; +#header { + height: 115px; background-image: url(header.png); } -#body{ - height:100%; - width:100%; +#body { + height: 100%; + width: 100%; background-image: url(body.png); - background-size:100% 100%; + background-size: 100% 100%; } -#footer{ - color:white; - height:70px; - line-height:70px; - text-align:middle; - clear:both; - text-align:center; +#footer { + color: white; + height: 70px; + line-height: 70px; + text-align: middle; + clear: both; + text-align: center; background-image: url(foot.png); } \ No newline at end of file diff --git a/examples/streaming/stockcrawler/src/main/resources/stock/js/stock.js b/examples/streaming/stockcrawler/src/main/resources/stock/js/stock.js index 4e5e5f6e8..97f1e0703 100644 --- a/examples/streaming/stockcrawler/src/main/resources/stock/js/stock.js +++ b/examples/streaming/stockcrawler/src/main/resources/stock/js/stock.js @@ -1,10 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + function initChart(chartid, tableid, stockId) { require.config({ paths: { - echarts: 'http://echarts.baidu.com/build/dist' + echarts: 'http://echarts.baidu.com/build/dist' } }); - + require( [ 'echarts', @@ -17,13 +35,13 @@ function initChart(chartid, tableid, stockId) { var dataPoints = 100; var timeTicket; clearInterval(timeTicket); - timeTicket = setInterval(function (){ - $.getJSON( "report/" + stockId, function( json ) { + timeTicket = setInterval(function () { + $.getJSON("report/" + stockId, function (json) { STOCK_NAME = json.name - + var maxDrawnDown = json.currentMax[0].max.price - json.currentMax[0].min.price; - var time = new Date(json.currentMax[0].current.timestamp).toLocaleTimeString().replace(/^\D*/,''); - // 动态数据接口 addData + var time = new Date(json.currentMax[0].current.timestamp).toLocaleTimeString().replace(/^\D*/, ''); + // 动态数据接口 addData myChart.addData([ [ 0, // 系列索引 @@ -40,91 +58,91 @@ function initChart(chartid, tableid, stockId) { time ] ]); - document.getElementById(chartid).style.display="block" + document.getElementById(chartid).style.display = "block" document.getElementById(tableid).innerHTML = "
" + JSON.stringify(json, null, 2) + "
" - }); + }); }, 2000); var subtext_ = "Draw Down" var option = { - title : { - text: 'Stock Analysis', - subtext: "Max " + subtext_ + title: { + text: 'Stock Analysis', + subtext: "Max " + subtext_ }, - tooltip : { - trigger: 'axis' + tooltip: { + trigger: 'axis' }, legend: { - data:["Current Price", "Current Draw Down"] + data: ["Current Price", "Current Draw Down"] }, toolbox: { - show : false, - feature : { - mark : {show: true}, - dataView : {show: true, readOnly: false}, - magicType : {show: true, type: ['line', 'bar']}, - restore : {show: true}, - saveAsImage : {show: true} + show: false, + feature: { + mark: {show: true}, + dataView: {show: true, readOnly: false}, + magicType: {show: true, type: ['line', 'bar']}, + restore: {show: true}, + saveAsImage: {show: true} } }, - dataZoom : { - show : false, - start : 0, - end : 100 + dataZoom: { + show: false, + start: 0, + end: 100 }, - xAxis : [ + xAxis: [ { - type : 'category', - boundaryGap : true, - data : (function (){ + type: 'category', + boundaryGap: true, + data: (function () { var now = new Date(); var res = []; var len = dataPoints; while (len--) { - res.unshift(now.toLocaleTimeString().replace(/^\D*/,'')); + res.unshift(now.toLocaleTimeString().replace(/^\D*/, '')); now = new Date(now - 2000); } return res; })() } ], - yAxis : [ + yAxis: [ { - type : 'value', + type: 'value', scale: true, - name : subtext_ + ' 价格/元', + name: subtext_ + ' 价格/元', boundaryGap: [0, 0.3] }, { - type : 'value', + type: 'value', scale: true, - name : 'Current 价格/元', + name: 'Current 价格/元', boundaryGap: [0, 0.1] } ], - series : [ + series: [ { - name:"Current Draw Down", - type:'line', - data:(function (){ + name: "Current Draw Down", + type: 'line', + data: (function () { var res = []; var len = dataPoints; while (len--) { - res.push(0); + res.push(0); } return res; })() }, { - name:"Current Price", - type:'line', + name: "Current Price", + type: 'line', yAxisIndex: 1, - data:(function (){ + data: (function () { var res = []; var len = dataPoints; while (len--) { - res.push(0); + res.push(0); } return res; })() diff --git a/examples/streaming/stockcrawler/src/main/resources/stock/stock.html b/examples/streaming/stockcrawler/src/main/resources/stock/stock.html index 224473fea..9682a5303 100644 --- a/examples/streaming/stockcrawler/src/main/resources/stock/stock.html +++ b/examples/streaming/stockcrawler/src/main/resources/stock/stock.html @@ -1,66 +1,87 @@ + + - - - - - - + + + + + +
-
-
-
Sorry, we couldn't complete the request. Please check your network connectivity or +
Sorry, we couldn't complete the request. Please + check your network connectivity or contact the administrator for assistance.
diff --git a/services/dashboard/widgets/metrics_period_switcher.js b/services/dashboard/widgets/metrics_period_switcher.js index b40aa2364..de9b62cdd 100644 --- a/services/dashboard/widgets/metrics_period_switcher.js +++ b/services/dashboard/widgets/metrics_period_switcher.js @@ -5,7 +5,7 @@ angular.module('dashboard') - .directive('metricsPeriodSwitcher', function() { + .directive('metricsPeriodSwitcher', function () { 'use strict'; return { @@ -15,7 +15,7 @@ angular.module('dashboard') pastHours: '=', viewCurrent: '=' }, - link: function(scope) { + link: function (scope) { 'use strict'; scope.options = { @@ -23,7 +23,7 @@ angular.module('dashboard') hist: 'Past %d Hours'.replace('%d', scope.pastHours) }; scope.value = scope.viewCurrent ? 'current' : 'hist'; - scope.$watch('value', function(value) { + scope.$watch('value', function (value) { scope.viewCurrent = String(value) === 'current'; }); } diff --git a/services/dashboard/widgets/radio_group.js b/services/dashboard/widgets/radio_group.js index aa5427d2a..cc467885f 100644 --- a/services/dashboard/widgets/radio_group.js +++ b/services/dashboard/widgets/radio_group.js @@ -5,7 +5,7 @@ angular.module('dashboard') - .directive('radioGroup', function() { + .directive('radioGroup', function () { 'use strict'; return { @@ -16,11 +16,11 @@ angular.module('dashboard') ngModel: '=', options: '=' }, - link: function(scope, elem, attrs) { + link: function (scope, elem, attrs) { 'use strict'; scope.buttonWidth = attrs.buttonWidth || '96px'; - scope.toggle = function(value) { + scope.toggle = function (value) { scope.ngModel = value; }; } diff --git a/services/js/src/main/scala/io/gearpump/dashboard/DashboardApp.scala b/services/js/src/main/scala/io/gearpump/dashboard/DashboardApp.scala index 51e6f0d51..3d80c4ee5 100644 --- a/services/js/src/main/scala/io/gearpump/dashboard/DashboardApp.scala +++ b/services/js/src/main/scala/io/gearpump/dashboard/DashboardApp.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/services/jvm/src/main/scala/io/gearpump/services/AdminService.scala b/services/jvm/src/main/scala/io/gearpump/services/AdminService.scala index aa88d1363..eb6531fc3 100644 --- a/services/jvm/src/main/scala/io/gearpump/services/AdminService.scala +++ b/services/jvm/src/main/scala/io/gearpump/services/AdminService.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,32 +18,34 @@ package io.gearpump.services -import akka.actor.{ActorSystem} +import akka.actor.ActorSystem import akka.http.scaladsl.model._ import akka.http.scaladsl.server.Directives._ -import akka.stream.{Materializer} -import io.gearpump.util.{Constants, Util} +import akka.stream.Materializer + +// NOTE: This cannot be removed!!! import io.gearpump.services.util.UpickleUtil._ + /** * AdminService is for cluster-wide managements. it is not related with * specific application. * * For example: - * Security management: Add user, remove user. - * Configuration management: Change configurations. - * Machine management: Add worker machines, remove worker machines, and add masters. + * - Security management: Add user, remove user. + * - Configuration management: Change configurations. + * - Machine management: Add worker machines, remove worker machines, and add masters. */ // TODO: Add YARN resource manager capacities to add/remove machines. class AdminService(override val system: ActorSystem) extends BasicService { - override def prefix = Neutral + protected override def prefix = Neutral - override def doRoute(implicit mat: Materializer) = { + protected override def doRoute(implicit mat: Materializer) = { path("terminate") { post { - system.shutdown() + system.terminate() complete(StatusCodes.NotFound) } } diff --git a/services/jvm/src/main/scala/io/gearpump/services/AppMasterService.scala b/services/jvm/src/main/scala/io/gearpump/services/AppMasterService.scala index 30ec0614f..060e78087 100644 --- a/services/jvm/src/main/scala/io/gearpump/services/AppMasterService.scala +++ b/services/jvm/src/main/scala/io/gearpump/services/AppMasterService.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,12 +18,16 @@ package io.gearpump.services -import akka.actor.{ActorSystem, ActorRef} +import scala.util.{Failure, Success, Try} + +import akka.actor.{ActorRef, ActorSystem} import akka.http.scaladsl.model.{FormData, Multipart} import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.Route import akka.http.scaladsl.server.directives.ParameterDirectives.ParamMagnet -import akka.stream.{Materializer, ActorMaterializer} +import akka.stream.Materializer +import upickle.default.{read, write} + import io.gearpump.cluster.AppMasterToMaster.{AppMasterSummary, GeneralAppMasterSummary} import io.gearpump.cluster.ClientToMaster._ import io.gearpump.cluster.ClusterConfig @@ -31,6 +35,8 @@ import io.gearpump.cluster.MasterToAppMaster.{AppMasterData, AppMasterDataDetail import io.gearpump.cluster.MasterToClient._ import io.gearpump.jarstore.JarStoreService import io.gearpump.services.AppMasterService.Status +// NOTE: This cannot be removed!!! +import io.gearpump.services.util.UpickleUtil._ import io.gearpump.streaming.AppMasterToMaster.StallingTasks import io.gearpump.streaming.appmaster.DagManager._ import io.gearpump.streaming.appmaster.StreamAppMasterSummary @@ -38,9 +44,6 @@ import io.gearpump.streaming.executor.Executor.{ExecutorConfig, ExecutorSummary, import io.gearpump.util.ActorUtil.{askActor, askAppMaster} import io.gearpump.util.FileDirective._ import io.gearpump.util.{Constants, Util} -import upickle.default.{read, write} -import io.gearpump.services.util.UpickleUtil._ -import scala.util.{Failure, Success, Try} /** * Management service for AppMaster @@ -52,40 +55,42 @@ class AppMasterService(val master: ActorRef, private val systemConfig = system.settings.config private val concise = systemConfig.getBoolean(Constants.GEARPUMP_SERVICE_RENDER_CONFIG_CONCISE) - override def doRoute(implicit mat: Materializer) = pathPrefix("appmaster" / IntNumber) { appId => - path("dynamicdag") { - parameters(ParamMagnet("args")) { args: String => - def replaceProcessor(dagOperation: DAGOperation): Route = { - onComplete(askAppMaster[DAGOperationResult](master, appId, dagOperation)) { - case Success(value) => - complete(write(value)) - case Failure(ex) => - failWith(ex) + protected override def doRoute(implicit mat: Materializer) = pathPrefix("appmaster" / IntNumber) { + appId => { + path("dynamicdag") { + parameters(ParamMagnet("args")) { args: String => + def replaceProcessor(dagOperation: DAGOperation): Route = { + onComplete(askAppMaster[DAGOperationResult](master, appId, dagOperation)) { + case Success(value) => + complete(write(value)) + case Failure(ex) => + failWith(ex) + } } - } - val msg = java.net.URLDecoder.decode(args) - val dagOperation = read[DAGOperation](msg) - (post & entity(as[Multipart.FormData])) { _ => - uploadFile { form => - val jar = form.getFile("jar").map(_.file) + val msg = java.net.URLDecoder.decode(args, "UTF-8") + val dagOperation = read[DAGOperation](msg) + (post & entity(as[Multipart.FormData])) { _ => + uploadFile { form => + val jar = form.getFile("jar").map(_.file) - if (jar.nonEmpty) { - dagOperation match { - case replace: ReplaceProcessor => - val description = replace.newProcessorDescription.copy(jar = Util.uploadJar(jar.get, jarStore)) - val dagOperationWithJar = replace.copy(newProcessorDescription = description) - replaceProcessor(dagOperationWithJar) + if (jar.nonEmpty) { + dagOperation match { + case replace: ReplaceProcessor => + val description = replace.newProcessorDescription.copy(jar = + Util.uploadJar(jar.get, jarStore)) + val dagOperationWithJar = replace.copy(newProcessorDescription = description) + replaceProcessor(dagOperationWithJar) + } + } else { + replaceProcessor(dagOperation) } - } else { - replaceProcessor(dagOperation) } + } ~ (post & entity(as[FormData])) { _ => + replaceProcessor(dagOperation) } - } ~ (post & entity(as[FormData])) { _ => - replaceProcessor(dagOperation) - } - } - } ~ + } + } ~ path("stallingtasks") { onComplete(askAppMaster[StallingTasks](master, appId, GetStallingTasks(appId))) { case Success(value) => @@ -124,23 +129,25 @@ class AppMasterService(val master: ActorRef, val executorId = Integer.parseInt(executorIdString) onComplete(askAppMaster[ExecutorConfig](master, appId, QueryExecutorConfig(executorId))) { case Success(value) => - val config = Option(value.config).map(ClusterConfig.render(_, concise)).getOrElse("{}") + val config = Option(value.config).map(ClusterConfig.render(_, concise)) + .getOrElse("{}") complete(config) case Failure(ex) => failWith(ex) } } ~ - pathEnd { - get { - val executorId = Integer.parseInt(executorIdString) - onComplete(askAppMaster[ExecutorSummary](master, appId, GetExecutorSummary(executorId))) { + pathEnd { + get { + val executorId = Integer.parseInt(executorIdString) + onComplete(askAppMaster[ExecutorSummary](master, appId, + GetExecutorSummary(executorId))) { case Success(value) => complete(write(value)) case Failure(ex) => failWith(ex) + } } } - } } ~ path("metrics" / RestPath) { path => parameterMap { optionMap => @@ -210,6 +217,7 @@ class AppMasterService(val master: ActorRef, } } } + } } } diff --git a/services/jvm/src/main/scala/io/gearpump/services/BasicService.scala b/services/jvm/src/main/scala/io/gearpump/services/BasicService.scala index 079afc3b2..75f033b1a 100644 --- a/services/jvm/src/main/scala/io/gearpump/services/BasicService.scala +++ b/services/jvm/src/main/scala/io/gearpump/services/BasicService.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,28 +18,31 @@ package io.gearpump.services -import akka.actor.{ActorSystem} +import scala.concurrent.ExecutionContext + +import akka.actor.ActorSystem import akka.http.scaladsl.model.headers.CacheDirectives.{`max-age`, `no-cache`} import akka.http.scaladsl.model.headers.`Cache-Control` import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.server.{Route} -import akka.stream.{Materializer} -import io.gearpump.util.{LogUtil, Constants} +import akka.http.scaladsl.server.Route +import akka.stream.Materializer -import scala.concurrent.ExecutionContext +import io.gearpump.util.{Constants, LogUtil} +// NOTE: This cannot be removed!!! +import io.gearpump.services.util.UpickleUtil._ trait RouteService { def route: Route } /** - * Wrap the cache behavior, and some common utils. + * Wraps the cache behavior, and some common utils. */ -trait BasicService extends RouteService{ +trait BasicService extends RouteService { implicit def system: ActorSystem - implicit def timeout = Constants.FUTURE_TIMEOUT + implicit def timeout: akka.util.Timeout = Constants.FUTURE_TIMEOUT implicit def ec: ExecutionContext = system.dispatcher @@ -53,7 +56,7 @@ trait BasicService extends RouteService{ private val noCacheHeader = `Cache-Control`(`no-cache`, `max-age`(0L)) def route: Route = encodeResponse { - extractMaterializer {implicit mat => + extractMaterializer { implicit mat => rawPathPrefix(prefix) { if (cache) { doRoute(mat) diff --git a/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala b/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala index 91f25fed5..6ca0f981e 100644 --- a/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala +++ b/services/jvm/src/main/scala/io/gearpump/services/MasterService.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,13 +16,15 @@ * limitations under the License. */ - package io.gearpump.services import java.io.{File, IOException} -import java.nio.file.Files import java.nio.charset.StandardCharsets.UTF_8 -import java.nio.file.StandardOpenOption.{WRITE, APPEND} +import java.nio.file.Files +import java.nio.file.StandardOpenOption.{APPEND, WRITE} +import scala.collection.JavaConverters._ +import scala.concurrent.Future +import scala.util.{Failure, Success} import akka.actor.{ActorRef, ActorSystem} import akka.http.scaladsl.server.Directives._ @@ -30,6 +32,7 @@ import akka.http.scaladsl.server.directives.ParameterDirectives.ParamMagnet import akka.http.scaladsl.unmarshalling.Unmarshaller._ import akka.stream.Materializer import com.typesafe.config.Config + import io.gearpump.cluster.AppMasterToMaster.{GetAllWorkers, GetMasterData, GetWorkerData, MasterData, WorkerData} import io.gearpump.cluster.ClientToMaster.{QueryHistoryMetrics, QueryMasterConfig, ReadOption} import io.gearpump.cluster.MasterToAppMaster.{AppMastersData, AppMastersDataRequest, WorkerList} @@ -38,18 +41,16 @@ import io.gearpump.cluster.client.ClientContext import io.gearpump.cluster.worker.WorkerSummary import io.gearpump.cluster.{ClusterConfig, UserConfig} import io.gearpump.jarstore.JarStoreService -import io.gearpump.services.MasterService.{BuiltinPartitioners, SubmitApplicationRequest} import io.gearpump.partitioner.{PartitionerByClassName, PartitionerDescription} +import io.gearpump.services.MasterService.{BuiltinPartitioners, SubmitApplicationRequest} +// NOTE: This cannot be removed!!! +import io.gearpump.services.util.UpickleUtil._ import io.gearpump.streaming.{ProcessorDescription, ProcessorId, StreamApplication} import io.gearpump.util.ActorUtil._ import io.gearpump.util.FileDirective._ import io.gearpump.util.{Constants, Graph, Util} -import io.gearpump.services.util.UpickleUtil._ - -import scala.collection.JavaConversions._ -import scala.concurrent.Future -import scala.util.{Failure, Success} +/** Manages service for master node */ class MasterService(val master: ActorRef, val jarStore: JarStoreService, override val system: ActorSystem) extends BasicService { @@ -59,7 +60,7 @@ class MasterService(val master: ActorRef, private val systemConfig = system.settings.config private val concise = systemConfig.getBoolean(Constants.GEARPUMP_SERVICE_RENDER_CONFIG_CONCISE) - override def doRoute(implicit mat: Materializer) = pathPrefix("master") { + protected override def doRoute(implicit mat: Materializer) = pathPrefix("master") { pathEnd { get { onComplete(askActor[MasterData](master, GetMasterData)) { @@ -76,16 +77,17 @@ class MasterService(val master: ActorRef, } } ~ path("workerlist") { - def future = askActor[WorkerList](master, GetAllWorkers).flatMap { workerList => - val workers = workerList.workers - val workerDataList = List.empty[WorkerSummary] - - Future.fold(workers.map { workerId => - askWorker[WorkerData](master, workerId, GetWorkerData(workerId)) - })(workerDataList) { (workerDataList, workerData) => - workerDataList :+ workerData.workerDescription + def future: Future[List[WorkerSummary]] = askActor[WorkerList](master, GetAllWorkers) + .flatMap { workerList => + val workers = workerList.workers + val workerDataList = List.empty[WorkerSummary] + + Future.fold(workers.map { workerId => + askWorker[WorkerData](master, workerId, GetWorkerData(workerId)) + })(workerDataList) { (workerDataList, workerData) => + workerDataList :+ workerData.workerDescription + } } - } onComplete(future) { case Success(result: List[WorkerSummary]) => complete(write(result)) case Failure(ex) => failWith(ex) @@ -221,9 +223,11 @@ object MasterService { case class Status(success: Boolean, reason: String = null) /** - * Submit Native Application. + * Submits Native Application. */ - def submitGearApp(jar: Option[File], executorNum: Int, args: String, systemConfig: Config, userConfigFile: Option[File]): Boolean = { + def submitGearApp( + jar: Option[File], executorNum: Int, args: String, + systemConfig: Config, userConfigFile: Option[File]): Boolean = { submitAndDeleteTempFiles( "io.gearpump.cluster.main.AppSubmitter", argsArray = Array("-executors", executorNum.toString) ++ spaceSeparatedArgumentsToArray(args), @@ -235,9 +239,10 @@ object MasterService { } /** - * Submit Storm application. + * Submits Storm application. */ - def submitStormApp(jar: Option[File], stormConf: Option[File], args: String, systemConfig: Config): Boolean = { + def submitStormApp( + jar: Option[File], stormConf: Option[File], args: String, systemConfig: Config): Boolean = { submitAndDeleteTempFiles( "io.gearpump.experiments.storm.main.GearpumpStormClient", argsArray = spaceSeparatedArgumentsToArray(args), @@ -248,9 +253,10 @@ object MasterService { ) } - private def submitAndDeleteTempFiles(mainClass: String, argsArray: Array[String], fileMap: Map[String, File], - classPath: Array[String], systemConfig: Config, - userConfigFile: Option[File] = None): Boolean = { + private def submitAndDeleteTempFiles( + mainClass: String, argsArray: Array[String], fileMap: Map[String, File], + classPath: Array[String], systemConfig: Config, + userConfigFile: Option[File] = None): Boolean = { try { val jar = fileMap.get("jar") if (jar.isEmpty) { @@ -279,7 +285,7 @@ object MasterService { } /** - * Return Java options for gearpump cluster + * Returns Java options for gearpump cluster */ private def clusterOptions(systemConfig: Config, userConfigFile: Option[File]): Array[String] = { var options = Array( @@ -288,7 +294,8 @@ object MasterService { s"-D${Constants.PREFER_IPV4}=true" ) - val masters = systemConfig.getStringList(Constants.GEARPUMP_CLUSTER_MASTERS).flatMap(Util.parseHostList) + val masters = systemConfig.getStringList(Constants.GEARPUMP_CLUSTER_MASTERS).asScala + .toList.flatMap(Util.parseHostList) options ++= masters.zipWithIndex.map { case (master, index) => s"-D${Constants.GEARPUMP_CLUSTER_MASTERS}.$index=${master.host}:${master.port}" }.toArray[String] @@ -311,7 +318,7 @@ object MasterService { } /** - * Return a space separated arguments as an array. + * Returns a space separated arguments as an array. */ private def spaceSeparatedArgumentsToArray(str: String): Array[String] = { str.split(" +").filter(_.nonEmpty) @@ -335,9 +342,9 @@ object MasterService { ) } - case class SubmitApplicationRequest ( - appName: String, - processors: Map[ProcessorId, ProcessorDescription], - dag: Graph[Int, String], - userconfig: UserConfig) + case class SubmitApplicationRequest( + appName: String, + processors: Map[ProcessorId, ProcessorDescription], + dag: Graph[Int, String], + userconfig: UserConfig) } diff --git a/services/jvm/src/main/scala/io/gearpump/services/RestServices.scala b/services/jvm/src/main/scala/io/gearpump/services/RestServices.scala index a44e0a935..d0fea30e0 100644 --- a/services/jvm/src/main/scala/io/gearpump/services/RestServices.scala +++ b/services/jvm/src/main/scala/io/gearpump/services/RestServices.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,20 +18,25 @@ package io.gearpump.services +import scala.concurrent.Await +import scala.concurrent.duration._ + import akka.actor.{ActorRef, ActorSystem} import akka.http.scaladsl.model.StatusCodes._ import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.{Route, _} import akka.stream.ActorMaterializer import akka.util.Timeout -import io.gearpump.jarstore.JarStoreService -import io.gearpump.util.{Constants, LogUtil} import org.apache.commons.lang.exception.ExceptionUtils -import scala.concurrent.Await -import scala.concurrent.duration._ +import io.gearpump.jarstore.JarStoreService +import io.gearpump.util.{Constants, LogUtil} +// NOTE: This cannot be removed!!! +import io.gearpump.services.util.UpickleUtil._ -class RestServices(master: ActorRef, mat: ActorMaterializer, system: ActorSystem) extends RouteService { +/** Contains all REST API service endpoints */ +class RestServices(master: ActorRef, mat: ActorMaterializer, system: ActorSystem) + extends RouteService { implicit val timeout = Constants.FUTURE_TIMEOUT @@ -42,9 +47,11 @@ class RestServices(master: ActorRef, mat: ActorMaterializer, system: ActorSystem private val LOG = LogUtil.getLogger(getClass) - private val securityEnabled = config.getBoolean(Constants.GEARPUMP_UI_SECURITY_AUTHENTICATION_ENABLED) + private val securityEnabled = config.getBoolean( + Constants.GEARPUMP_UI_SECURITY_AUTHENTICATION_ENABLED) - private val supervisorPath = system.settings.config.getString(Constants.GEARPUMP_SERVICE_SUPERVISOR_PATH) + private val supervisorPath = system.settings.config.getString( + Constants.GEARPUMP_SERVICE_SUPERVISOR_PATH) private val myExceptionHandler: ExceptionHandler = ExceptionHandler { case ex: Throwable => { @@ -55,7 +62,7 @@ class RestServices(master: ActorRef, mat: ActorMaterializer, system: ActorSystem } } - // make sure staticRoute is the final one, as it will try to lookup resource in local path + // Makes sure staticRoute is the final one, as it will try to lookup resource in local path // if there is no match in previous routes private val static = new StaticService(system, supervisorPath).route diff --git a/services/jvm/src/main/scala/io/gearpump/services/SecurityService.scala b/services/jvm/src/main/scala/io/gearpump/services/SecurityService.scala index a5bbe2f10..b1abf62dd 100644 --- a/services/jvm/src/main/scala/io/gearpump/services/SecurityService.scala +++ b/services/jvm/src/main/scala/io/gearpump/services/SecurityService.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,54 +18,57 @@ package io.gearpump.services -import akka.actor.{ActorSystem} -import akka.http.scaladsl.model.{Uri, StatusCodes, RemoteAddress} -import akka.http.scaladsl.model.headers.{HttpCookiePair, HttpCookie, HttpChallenge} -import akka.http.scaladsl.server.AuthenticationFailedRejection.{CredentialsRejected, CredentialsMissing} -import akka.http.scaladsl.server._ +import scala.concurrent.{ExecutionContext, Future} +import scala.util.{Failure, Success, Try} + +import akka.actor.ActorSystem +import akka.http.scaladsl.model.headers.{HttpChallenge, HttpCookie, HttpCookiePair} +import akka.http.scaladsl.model.{RemoteAddress, StatusCodes, Uri} +import akka.http.scaladsl.server.AuthenticationFailedRejection.{CredentialsMissing, CredentialsRejected} import akka.http.scaladsl.server.Directives._ +import akka.http.scaladsl.server._ import akka.http.scaladsl.server.directives.FormFieldDirectives.FieldMagnet import akka.stream.Materializer -import com.softwaremill.session.ClientSessionManagerMagnet._ -import com.softwaremill.session.SessionDirectives._ -import com.softwaremill.session._ import com.typesafe.config.Config +import com.softwaremill.session.SessionDirectives._ +import com.softwaremill.session.SessionOptions._ +import com.softwaremill.session.{MultiValueSessionSerializer, SessionConfig, SessionManager} +import upickle.default.write + +import io.gearpump.security.{Authenticator => BaseAuthenticator} import io.gearpump.services.SecurityService.{User, UserSession} import io.gearpump.services.security.oauth2.OAuth2Authenticator import io.gearpump.util.{Constants, LogUtil} -import upickle.default.{write} +// NOTE: This cannot be removed!!! import io.gearpump.services.util.UpickleUtil._ -import io.gearpump.security.{Authenticator => BaseAuthenticator} -import scala.concurrent.{ExecutionContext, Future} -import scala.util.{Failure, Success} - /** - * When user cannot be authenticated, will reject with 401 AuthenticationFailedRejection - * When user can be authenticated, but are not authorized to access certail resource, will - * return a 405 AuthorizationFailedRejection. - * - * When web UI frontend receive 401, it should redirect the UI to login page. - * When web UI receive 405,it should display errors like - * "current user is not authorized to access this resource." + * Security authentication endpoint. * - * The Authenticator used is pluggable, the current Authenticator is resolved by looking up config path - * [[Constants.GEARPUMP_UI_AUTHENTICATOR_CLASS]]. + * - When user cannot be authenticated, will reject with 401 AuthenticationFailedRejection + * - When user can be authenticated, but are not authorized to access certail resource, will + * return a 405 AuthorizationFailedRejection. + * - When web UI frontend receive 401, it should redirect the UI to login page. + * - When web UI receive 405,it should display errors like + * "current user is not authorized to access this resource." * - * see [[BaseAuthenticator]] to find more info on custom Authenticator. + * The Authenticator used is pluggable, the current Authenticator is resolved by looking up + * config path [[io.gearpump.util.Constants.GEARPUMP_UI_AUTHENTICATOR_CLASS]]. * + * See [[io.gearpump.security.Authenticator]] to find more info on custom Authenticator. */ class SecurityService(inner: RouteService, implicit val system: ActorSystem) extends RouteService { // Use scheme "GearpumpBasic" to avoid popping up web browser native authentication box. - private val challenge = HttpChallenge(scheme = "GearpumpBasic", realm = "gearpump", params = Map.empty) + private val challenge = HttpChallenge(scheme = "GearpumpBasic", realm = "gearpump", + params = Map.empty) val LOG = LogUtil.getLogger(getClass, "AUDIT") private val config = system.settings.config private val sessionConfig = SessionConfig.fromConfig(config) - private implicit val sessionManager = new SessionManager[UserSession](sessionConfig) - private val magnet = ClientSessionManagerMagnet.forSessionManager[UserSession, Unit](Unit) + private implicit val sessionManager: SessionManager[UserSession] = + new SessionManager[UserSession](sessionConfig) private val authenticator = { val clazz = Class.forName(config.getString(Constants.GEARPUMP_UI_AUTHENTICATOR_CLASS)) @@ -74,7 +77,7 @@ class SecurityService(inner: RouteService, implicit val system: ActorSystem) ext authenticator } - private def configToMap(config : Config, path: String) = { + private def configToMap(config: Config, path: String) = { import scala.collection.JavaConverters._ config.getConfig(path).root.unwrapped.asScala.toMap map { case (k, v) => k -> v.toString } } @@ -91,8 +94,9 @@ class SecurityService(inner: RouteService, implicit val system: ActorSystem) ext } } - private def authenticate(user: String, pass: String)(implicit ec: ExecutionContext): Future[Option[UserSession]] = { - authenticator.authenticate(user, pass, ec).map{ result => + private def authenticate(user: String, pass: String)(implicit ec: ExecutionContext) + : Future[Option[UserSession]] = { + authenticator.authenticate(user, pass, ec).map { result => if (result.authenticated) { Some(UserSession(user, result.permissionLevel)) } else { @@ -110,7 +114,7 @@ class SecurityService(inner: RouteService, implicit val system: ActorSystem) ext } private def requireAuthentication(inner: UserSession => Route): Route = { - optionalSession(magnet) { sessionOption => + optionalSession(oneOff, usingCookiesOrHeaders) { sessionOption => sessionOption match { case Some(session) => { inner(session) @@ -122,10 +126,12 @@ class SecurityService(inner: RouteService, implicit val system: ActorSystem) ext } private def login(session: UserSession, ip: String, redirectToRoot: Boolean = false): Route = { - setSession(session) { + setSession(oneOff, usingCookies, session) { val user = session.user - val maxAgeMs = 1000 * sessionConfig.clientSessionMaxAgeSeconds.getOrElse(24 * 3600L) // default 1 day - setCookie(HttpCookie.fromPair(HttpCookiePair("username", user), path = Some("/"), maxAge = Some(maxAgeMs))) { + // Default: 1 day + val maxAgeMs = 1000 * sessionConfig.sessionMaxAgeSeconds.getOrElse(24 * 3600L) + setCookie(HttpCookie.fromPair(HttpCookiePair("username", user), path = Some("/"), + maxAge = Some(maxAgeMs))) { LOG.info(s"user $user login from $ip") if (redirectToRoot) { redirect(Uri("/"), StatusCodes.TemporaryRedirect) @@ -137,7 +143,7 @@ class SecurityService(inner: RouteService, implicit val system: ActorSystem) ext } private def logout(user: UserSession, ip: String): Route = { - invalidateSession(magnet) { ctx => + invalidateSession(oneOff, usingCookies) { ctx => LOG.info(s"user ${user.user} logout from $ip") ctx.complete(write(new User(user.user))) } @@ -145,13 +151,13 @@ class SecurityService(inner: RouteService, implicit val system: ActorSystem) ext // Only admin are able to access operation like post/delete/put private def requireAuthorization(user: UserSession, route: => Route): Route = { - // valid user + // Valid user if (user.permissionLevel >= BaseAuthenticator.User.permissionLevel) { route } else { - // possibly a guest or not authenticated. + // Possibly a guest or not authenticated. (put | delete | post) { - // reject with 405 authorization error + // Reject with 405 authorization error reject(AuthorizationFailedRejection) } ~ get { @@ -171,8 +177,8 @@ class SecurityService(inner: RouteService, implicit val system: ActorSystem) ext extractExecutionContext{implicit ec: ExecutionContext => extractMaterializer{implicit mat: Materializer => (extractClientIP | unknownIp) { ip => - pathPrefix("login") { - pathEndOrSingleSlash { + pathPrefix("login") { + pathEndOrSingleSlash { get { getFromResource("login/login.html") } ~ @@ -181,7 +187,7 @@ class SecurityService(inner: RouteService, implicit val system: ActorSystem) ext formField(FieldMagnet('username.as[String])) {user: String => formFields(FieldMagnet('password.as[String])) {pass: String => val result = authenticate(user, pass) - onSuccess(result){ + onSuccess(result) { case Some(session) => login(session, ip.toString) case None => @@ -192,7 +198,7 @@ class SecurityService(inner: RouteService, implicit val system: ActorSystem) ext } } ~ path ("oauth2" / "providers") { - // respond with a list of OAuth2 providers. + // Responds with a list of OAuth2 providers. complete(write(oauth2Providers)) } ~ // Support OAUTH Authentication @@ -207,7 +213,7 @@ class SecurityService(inner: RouteService, implicit val system: ActorSystem) ext def loginWithOAuth2Parameters(parameters: Map[String, String]): Route = { val result = oauthService.authenticate(parameters) - onComplete(result){ + onComplete(result) { case Success(session) => login(session, ip.toString, redirectToRoot = true) case Failure(ex) => { @@ -218,7 +224,7 @@ class SecurityService(inner: RouteService, implicit val system: ActorSystem) ext } path ("authorize") { - // redirect to OAuth2 service provider for authorization. + // Redirects to OAuth2 service provider for authorization. redirect(Uri(oauthService.getAuthorizationUrl), StatusCodes.TemporaryRedirect) } ~ path ("accesstoken") { @@ -255,17 +261,28 @@ class SecurityService(inner: RouteService, implicit val system: ActorSystem) ext object SecurityService { - val SESSION_MANAGER_KEY = "akka.http.session.serverSecret" + val SESSION_MANAGER_KEY = "akka.http.session.server-secret" case class UserSession(user: String, permissionLevel: Int) object UserSession { - implicit def serializer: SessionSerializer[UserSession] = new ToMapSessionSerializer[UserSession] { - private val User = "user" - private val PermissionLevel = "permissionLevel" - override def serializeToMap(t: UserSession) = Map(User -> t.user, PermissionLevel->t.permissionLevel.toString) - override def deserializeFromMap(m: Map[String, String]) = UserSession(m(User), m(PermissionLevel).toInt) + private val User = "user" + private val PermissionLevel = "permissionLevel" + + implicit def serializer: MultiValueSessionSerializer[UserSession] = { + new MultiValueSessionSerializer[UserSession]( + toMap = {t: UserSession => + Map(User -> t.user, PermissionLevel -> t.permissionLevel.toString) + }, + fromMap = {m: Map[String, String] => + if (m.contains(User)) { + Try(UserSession(m(User), m(PermissionLevel).toInt)) + } else { + Failure[UserSession](new Exception("Fail to parse session ")) + } + } + ) } } diff --git a/services/jvm/src/main/scala/io/gearpump/services/StaticService.scala b/services/jvm/src/main/scala/io/gearpump/services/StaticService.scala index 09180c3ca..fc990a1d5 100644 --- a/services/jvm/src/main/scala/io/gearpump/services/StaticService.scala +++ b/services/jvm/src/main/scala/io/gearpump/services/StaticService.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,11 +18,13 @@ package io.gearpump.services -import akka.actor.{ActorSystem} +import akka.actor.ActorSystem import akka.http.scaladsl.model._ import akka.http.scaladsl.server.Directives._ -import akka.stream.{Materializer} -import io.gearpump.util.{Constants, Util} +import akka.stream.Materializer + +import io.gearpump.util.Util +// NOTE: This cannot be removed!!! import io.gearpump.services.util.UpickleUtil._ /** @@ -33,13 +35,13 @@ class StaticService(override val system: ActorSystem, supervisorPath: String) private val version = Util.version - override def prefix = Neutral + protected override def prefix = Neutral - override def cache = true + override def cache: Boolean = true - override def doRoute(implicit mat: Materializer) = { + protected override def doRoute(implicit mat: Materializer) = { path("version") { - get {ctx => + get { ctx => ctx.complete(version) } } ~ diff --git a/services/jvm/src/main/scala/io/gearpump/services/SupervisorService.scala b/services/jvm/src/main/scala/io/gearpump/services/SupervisorService.scala index 5d552d656..c7f19e4e5 100644 --- a/services/jvm/src/main/scala/io/gearpump/services/SupervisorService.scala +++ b/services/jvm/src/main/scala/io/gearpump/services/SupervisorService.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,39 +18,43 @@ package io.gearpump.services +import scala.concurrent.Future +import scala.util.{Failure, Success} + import akka.actor.{ActorRef, ActorSystem} import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.Route import akka.stream.Materializer -import io.gearpump.WorkerId -import io.gearpump.cluster.AppMasterToMaster.{GetWorkerData, WorkerData, GetAllWorkers} + +import io.gearpump.cluster.AppMasterToMaster.{GetWorkerData, WorkerData} import io.gearpump.cluster.ClientToMaster._ +import io.gearpump.cluster.worker.WorkerId import io.gearpump.services.SupervisorService.{Path, Status} import io.gearpump.util.ActorUtil._ - -import scala.concurrent.Future -import scala.util.{Failure, Success} -import upickle.default.{read, write} +// NOTE: This cannot be removed!!! import io.gearpump.services.util.UpickleUtil._ -class SupervisorService(val master: ActorRef, val supervisor: ActorRef, override val system: ActorSystem) +/** Responsible for adding/removing machines. Typically it delegates to YARN. */ +class SupervisorService( + val master: ActorRef, val supervisor: ActorRef, override val system: ActorSystem) extends BasicService { import upickle.default.write /** - * TODO: Add additional check to ensure the user have enough authorization to add/remove a worker machine + * TODO: Add additional check to ensure the user have enough authorization to + * add/remove a worker machine */ private def authorize(internal: Route): Route = { if (supervisor == null) { - failWith(new Exception("API not enabled, cannot find a valid supervisor! Please make sure Gearpump is " + - "running on top of YARN or other resource managers")) + failWith(new Exception("API not enabled, cannot find a valid supervisor! " + + "Please make sure Gearpump is running on top of YARN or other resource managers")) } else { internal } } - override def doRoute(implicit mat: Materializer) = pathPrefix("supervisor") { + protected override def doRoute(implicit mat: Materializer) = pathPrefix("supervisor") { pathEnd { get { val path = if (supervisor == null) { @@ -105,7 +109,7 @@ class SupervisorService(val master: ActorRef, val supervisor: ActorRef, override } } -object SupervisorService{ +object SupervisorService { case class Status(enabled: Boolean) case class Path(path: String) diff --git a/services/jvm/src/main/scala/io/gearpump/services/WorkerService.scala b/services/jvm/src/main/scala/io/gearpump/services/WorkerService.scala index a1edbe4b9..78cdb375f 100644 --- a/services/jvm/src/main/scala/io/gearpump/services/WorkerService.scala +++ b/services/jvm/src/main/scala/io/gearpump/services/WorkerService.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,20 +18,23 @@ package io.gearpump.services -import akka.actor.{ActorSystem, ActorRef} +import scala.util.{Failure, Success} + +import akka.actor.{ActorRef, ActorSystem} import akka.http.scaladsl.server.Directives._ -import akka.stream.{Materializer} -import io.gearpump.WorkerId +import akka.stream.Materializer + import io.gearpump.cluster.AppMasterToMaster.{GetWorkerData, WorkerData} -import io.gearpump.cluster.ClientToMaster.{ReadOption, QueryHistoryMetrics, QueryWorkerConfig} +import io.gearpump.cluster.ClientToMaster.{QueryHistoryMetrics, QueryWorkerConfig, ReadOption} import io.gearpump.cluster.ClusterConfig import io.gearpump.cluster.MasterToClient.{HistoryMetrics, WorkerConfig} +import io.gearpump.cluster.worker.WorkerId import io.gearpump.util.ActorUtil._ import io.gearpump.util.Constants +// NOTE: This cannot be removed!!! import io.gearpump.services.util.UpickleUtil._ -import scala.util.{Failure, Success} - +/** Service to handle worker related queries */ class WorkerService(val master: ActorRef, override val system: ActorSystem) extends BasicService { @@ -39,15 +42,17 @@ class WorkerService(val master: ActorRef, override val system: ActorSystem) private val systemConfig = system.settings.config private val concise = systemConfig.getBoolean(Constants.GEARPUMP_SERVICE_RENDER_CONFIG_CONCISE) - override def doRoute(implicit mat: Materializer) = pathPrefix("worker" / Segment) { workerIdString => - pathEnd { - val workerId = WorkerId.parse(workerIdString) - onComplete(askWorker[WorkerData](master, workerId, GetWorkerData(workerId))) { - case Success(value: WorkerData) => - complete(write(value.workerDescription)) - case Failure(ex) => failWith(ex) + protected override def doRoute(implicit mat: Materializer) = pathPrefix("worker" / Segment) { + workerIdString => { + pathEnd { + val workerId = WorkerId.parse(workerIdString) + onComplete(askWorker[WorkerData](master, workerId, GetWorkerData(workerId))) { + case Success(value: WorkerData) => + complete(write(value.workerDescription)) + case Failure(ex) => failWith(ex) + } } - } ~ + }~ path("config") { val workerId = WorkerId.parse(workerIdString) onComplete(askWorker[WorkerConfig](master, workerId, QueryWorkerConfig(workerId))) { diff --git a/services/jvm/src/main/scala/io/gearpump/services/main/Services.scala b/services/jvm/src/main/scala/io/gearpump/services/main/Services.scala index 7dc0277f2..a647a8d34 100644 --- a/services/jvm/src/main/scala/io/gearpump/services/main/Services.scala +++ b/services/jvm/src/main/scala/io/gearpump/services/main/Services.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,21 +18,26 @@ package io.gearpump.services.main -import akka.actor.{ActorSystem} +import java.util.Random +import scala.collection.JavaConverters._ +import scala.concurrent.Await + +import akka.actor.ActorSystem import akka.http.scaladsl.Http import akka.http.scaladsl.server.Route import akka.stream.ActorMaterializer import com.typesafe.config.ConfigValueFactory +import org.slf4j.Logger +import sun.misc.BASE64Encoder + import io.gearpump.cluster.ClusterConfig -import io.gearpump.cluster.main.{Gear, CLIOption, ArgumentsParser} +import io.gearpump.cluster.main.{ArgumentsParser, CLIOption, Gear} import io.gearpump.cluster.master.MasterProxy -import io.gearpump.services.{SecurityService, RestServices} +import io.gearpump.services.{RestServices, SecurityService} import io.gearpump.util.LogUtil.ProcessType import io.gearpump.util.{AkkaApp, Constants, LogUtil, Util} -import org.slf4j.Logger - -import scala.concurrent.Await +/** Command line to start UI server */ object Services extends AkkaApp with ArgumentsParser { private val LOG = LogUtil.getLogger(getClass) @@ -48,11 +53,13 @@ object Services extends AkkaApp with ArgumentsParser { ClusterConfig.ui() } - override def help: Unit = { + override def help(): Unit = { + // scalastyle:off println Console.println("UI Server") + // scalastyle:on println } - private var killFunction: Option[() =>Unit] = None + private var killFunction: Option[() => Unit] = None override def main(inputAkkaConf: Config, args: Array[String]): Unit = { @@ -69,48 +76,59 @@ object Services extends AkkaApp with ArgumentsParser { LogUtil.getLogger(getClass) } - - import scala.collection.JavaConversions._ if (argConfig.exists("master")) { val master = argConfig.getString("master") akkaConf = akkaConf.withValue(Constants.GEARPUMP_CLUSTER_MASTERS, - ConfigValueFactory.fromIterable(List(master))) + ConfigValueFactory.fromIterable(List(master).asJava)) } akkaConf = akkaConf.withValue(Constants.GEARPUMP_SERVICE_SUPERVISOR_PATH, ConfigValueFactory.fromAnyRef(argConfig.getString("supervisor"))) - // Create a random unique secret key for session manager. + // Creates a random unique secret key for session manager. // All previous stored session token cookies will be invalidated when UI // server is restarted. .withValue(SecurityService.SESSION_MANAGER_KEY, - ConfigValueFactory.fromAnyRef(java.util.UUID.randomUUID().toString())) + ConfigValueFactory.fromAnyRef(randomSeverSecret())) - val masterCluster = akkaConf.getStringList(Constants.GEARPUMP_CLUSTER_MASTERS).toList.flatMap(Util.parseHostList) + val masterCluster = akkaConf.getStringList(Constants.GEARPUMP_CLUSTER_MASTERS).asScala + .flatMap(Util.parseHostList) - implicit val system = ActorSystem("services" , akkaConf) + implicit val system = ActorSystem("services", akkaConf) implicit val executionContext = system.dispatcher import scala.concurrent.duration._ - val master = system.actorOf(MasterProxy.props(masterCluster, 1 day), s"masterproxy${system.name}") + val master = system.actorOf(MasterProxy.props(masterCluster, 1.day), + s"masterproxy${system.name}") val (host, port) = parseHostPort(system.settings.config) - implicit val mat = ActorMaterializer() val services = new RestServices(master, mat, system) val bindFuture = Http().bindAndHandle(Route.handlerFlow(services.route), host, port) - Await.result(bindFuture, 15 seconds) + Await.result(bindFuture, 15.seconds) - val displayHost = if(host == "0.0.0.0") "127.0.0.1" else host + val displayHost = if (host == "0.0.0.0") "127.0.0.1" else host LOG.info(s"Please browse to http://$displayHost:$port to see the web UI") + + // scalastyle:off println println(s"Please browse to http://$displayHost:$port to see the web UI") + // scalastyle:on println - killFunction = Some{() => + killFunction = Some { () => LOG.info("Shutting down UI Server") - system.shutdown() + system.terminate() } - system.awaitTermination() + Await.result(system.whenTerminated, Duration.Inf) + } + + private def randomSeverSecret(): String = { + val random = new Random() + val length = 64 // Required + val bytes = new Array[Byte](length) + random.nextBytes(bytes) + val endecoder = new BASE64Encoder() + endecoder.encode(bytes) } private def parseHostPort(config: Config): (String, Int) = { @@ -120,7 +138,8 @@ object Services extends AkkaApp with ArgumentsParser { } // TODO: fix this - // Hack around for YARN module, so that we can kill the UI server when application is shutting down. + // Hacks around for YARN module, so that we can kill the UI server + // when application is shutting down. def kill(): Unit = { if (killFunction.isDefined) { killFunction.get.apply() diff --git a/services/jvm/src/main/scala/io/gearpump/services/package.scala b/services/jvm/src/main/scala/io/gearpump/services/package.scala index bebcf95b8..7b38134a8 100644 --- a/services/jvm/src/main/scala/io/gearpump/services/package.scala +++ b/services/jvm/src/main/scala/io/gearpump/services/package.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/services/jvm/src/main/scala/io/gearpump/services/security/oauth2/OAuth2Authenticator.scala b/services/jvm/src/main/scala/io/gearpump/services/security/oauth2/OAuth2Authenticator.scala index edaefa782..158d406c0 100644 --- a/services/jvm/src/main/scala/io/gearpump/services/security/oauth2/OAuth2Authenticator.scala +++ b/services/jvm/src/main/scala/io/gearpump/services/security/oauth2/OAuth2Authenticator.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,11 +18,13 @@ package io.gearpump.services.security.oauth2 +import scala.concurrent.{ExecutionContext, Future} + import com.typesafe.config.Config + import io.gearpump.services.SecurityService.UserSession import io.gearpump.util.Constants import io.gearpump.util.Constants._ -import scala.concurrent.{ExecutionContext, Future} /** * @@ -30,46 +32,47 @@ import scala.concurrent.{ExecutionContext, Future} * @see [[https://tools.ietf.org/html/rfc6749]] to find what is OAuth2, and how it works. * * Basically flow for OAuth2 Authentication: - * 1. User accesses Gearpump UI website, and choose to login with OAuth2 server. - * 2. Gearpump UI website redirects user to OAuth2 server domain authorization endpoint. - * 3. End user complete the authorization in the domain of OAuth2 server. - * 4. OAuth2 server redirects user back to Gearpump UI server. - * 5. Gearpump UI server verify the tokens and extract credentials from query - * parameters and form fields. + * 1. User accesses Gearpump UI website, and choose to login with OAuth2 server. + * 2. Gearpump UI website redirects user to OAuth2 server domain authorization endpoint. + * 3. End user complete the authorization in the domain of OAuth2 server. + * 4. OAuth2 server redirects user back to Gearpump UI server. + * 5. Gearpump UI server verify the tokens and extract credentials from query + * parameters and form fields. * - * @note '''Thread-safety''' is a MUST requirement. Developer need to ensure the sub-class is thread-safe. - * Sub-class should have a parameterless constructor. + * NOTE: '''Thread-safety''' is a MUST requirement. Developer need to ensure the sub-class is + * thread-safe. Sub-class should have a parameterless constructor. * - * @note OAuth2 Authenticator requires access of Internet. Please make sure HTTP proxy are + * NOTE: OAuth2 Authenticator requires access of Internet. Please make sure HTTP proxy are * set properly if applied. * - * @example Config proxy when UI server is started on Windows: + * Example: Config proxy when UI server is started on Windows: * {{{ - * > set JAVA_OPTS=-Dhttp.proxyHost=xx.com -Dhttp.proxyPort=8088 -Dhttps.proxyHost=xx.com -Dhttps.proxyPort=8088 + * > set JAVA_OPTS=-Dhttp.proxyHost=xx.com -Dhttp.proxyPort=8088 -Dhttps.proxyHost=xx.com + * -Dhttps.proxyPort=8088 * > bin\services * }}} * - * @example Config proxy when UI server is started on Linux: + * Example: Config proxy when UI server is started on Linux: * {{{ - * $ export JAVA_OPTS="-Dhttp.proxyHost=xx.com -Dhttp.proxyPort=8088 -Dhttps.proxyHost=xx.com -Dhttps.proxyPort=8088" + * $ export JAVA_OPTS="-Dhttp.proxyHost=xx.com -Dhttp.proxyPort=8088 -Dhttps.proxyHost=xx.com + * -Dhttps.proxyPort=8088" * $ bin/services * }}} - * */ trait OAuth2Authenticator { /** * Inits authenticator with config which contains client ID, client secret, and etc.. * - * Typically, the client key and client secret is provided by OAuth2 Authorization server when user - * register an application there. - * @see [[https://tools.ietf.org/html/rfc6749]] for definition of client, client Id, + * Typically, the client key and client secret is provided by OAuth2 Authorization server + * when user register an application there. + * See [[https://tools.ietf.org/html/rfc6749]] for definition of client, client Id, * and client secret. * * See [[https://developer.github.com/v3/oauth/]] for an actual example of how Github * use client key, and client secret. * - * @note '''Thread-Safety''': Framework ensures this call is synchronized. + * NOTE: '''Thread-Safety''': Framework ensures this call is synchronized. * * @param config Client Id, client secret, callback URL and etc.. * @param executionContext ExecutionContext from hosting environment. @@ -80,7 +83,7 @@ trait OAuth2Authenticator { * Returns the OAuth Authorization URL so for redirection to that address to do OAuth2 * authorization. * - * @note '''Thread-Safety''': This can be called in a multi-thread environment. Developer + * NOTE: '''Thread-Safety''': This can be called in a multi-thread environment. Developer * need to ensure thread safety. */ def getAuthorizationUrl: String @@ -89,9 +92,11 @@ trait OAuth2Authenticator { * After authorization, OAuth2 server redirects user back with tokens. This verify the * tokens, retrieve the profiles, and return [[UserSession]] information. * - * @note This is an Async call. - * @note This call requires external internet access. - * @note '''Thread-Safety''': This can be called in a multi-thread environment. Developer + * NOTE: This is an Async call. + * + * NOTE: This call requires external internet access. + * + * NOTE: '''Thread-Safety''': This can be called in a multi-thread environment. Developer * need to ensure thread safety. * * @param parameters HTTP Query and Post parameters, which typically contains Authorization code. @@ -100,7 +105,7 @@ trait OAuth2Authenticator { def authenticate(parameters: Map[String, String]): Future[UserSession] /** - * Clean resource + * Clean up resource */ def close(): Unit } @@ -116,7 +121,8 @@ object OAuth2Authenticator { * @param provider, Name for the OAuth2 Authentication Service. * @return Returns null if the OAuth2 Authentication is disabled. */ - def get(config: Config, provider: String, executionContext: ExecutionContext): OAuth2Authenticator = { + def get(config: Config, provider: String, executionContext: ExecutionContext) + : OAuth2Authenticator = { if (providers.contains(provider)) { providers(provider) @@ -129,7 +135,8 @@ object OAuth2Authenticator { providers(provider) } else { val authenticatorConfig = config.getConfig(path) - val authenticatorClass = authenticatorConfig.getString(GEARPUMP_UI_OAUTH2_AUTHENTICATOR_CLASS) + val authenticatorClass = authenticatorConfig.getString( + GEARPUMP_UI_OAUTH2_AUTHENTICATOR_CLASS) val clazz = Thread.currentThread().getContextClassLoader.loadClass(authenticatorClass) val authenticator = clazz.newInstance().asInstanceOf[OAuth2Authenticator] authenticator.init(authenticatorConfig, executionContext) diff --git a/services/jvm/src/main/scala/io/gearpump/services/security/oauth2/impl/BaseOAuth2Authenticator.scala b/services/jvm/src/main/scala/io/gearpump/services/security/oauth2/impl/BaseOAuth2Authenticator.scala index f0ba4007c..e422d3fe5 100644 --- a/services/jvm/src/main/scala/io/gearpump/services/security/oauth2/impl/BaseOAuth2Authenticator.scala +++ b/services/jvm/src/main/scala/io/gearpump/services/security/oauth2/impl/BaseOAuth2Authenticator.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,14 +19,18 @@ package io.gearpump.services.security.oauth2.impl import java.util.concurrent.atomic.AtomicBoolean +import scala.collection.mutable.StringBuilder +import scala.concurrent.{ExecutionContext, Future, Promise} +import scala.util.{Failure, Success} +import com.typesafe.config.Config import com.github.scribejava.core.builder.ServiceBuilderAsync import com.github.scribejava.core.builder.api.DefaultApi20 import com.github.scribejava.core.model._ import com.github.scribejava.core.oauth.OAuth20Service import com.github.scribejava.core.utils.OAuthEncoder import com.ning.http.client.AsyncHttpClientConfig -import com.typesafe.config.Config + import io.gearpump.security.Authenticator import io.gearpump.services.SecurityService.UserSession import io.gearpump.services.security.oauth2.OAuth2Authenticator @@ -34,14 +38,10 @@ import io.gearpump.services.security.oauth2.impl.BaseOAuth2Authenticator.BaseApi import io.gearpump.util.Constants._ import io.gearpump.util.Util -import scala.collection.mutable.StringBuilder -import scala.concurrent.{ExecutionContext, Future, Promise} -import scala.util.{Failure, Success} - /** * Uses Ning AsyncClient to connect to OAuth2 service. * - * @see [[OAuth2Authenticator]] for more API information. + * See [[io.gearpump.services.security.oauth2.OAuth2Authenticator]] for more API information. */ abstract class BaseOAuth2Authenticator extends OAuth2Authenticator { @@ -105,7 +105,6 @@ abstract class BaseOAuth2Authenticator extends OAuth2Authenticator { oauthService.getAuthorizationUrl() } - protected def authenticateWithAccessToken(accessToken: OAuth2AccessToken): Future[UserSession] = { val promise = Promise[UserSession]() val request = new OAuthRequestAsync(Verb.GET, protectedResourceUrl, oauthService) @@ -139,7 +138,7 @@ abstract class BaseOAuth2Authenticator extends OAuth2Authenticator { new OAuthAsyncRequestCallback[OAuth2AccessToken] { override def onCompleted(accessToken: OAuth2AccessToken): Unit = { - authenticateWithAccessToken(accessToken).onComplete{ + authenticateWithAccessToken(accessToken).onComplete { case Success(user) => promise.success(user) case Failure(ex) => promise.failure(ex) } @@ -167,9 +166,11 @@ abstract class BaseOAuth2Authenticator extends OAuth2Authenticator { } } - private def buildOAuth2Service(clientId: String, clientSecret: String, callback: String): OAuth20Service = { - val state: String = "state" + Util.randInt - ScribeJavaConfig.setForceTypeOfHttpRequests(ForceTypeOfHttpRequest.FORCE_ASYNC_ONLY_HTTP_REQUESTS) + private def buildOAuth2Service(clientId: String, clientSecret: String, callback: String) + : OAuth20Service = { + val state: String = "state" + Util.randInt() + ScribeJavaConfig.setForceTypeOfHttpRequests( + ForceTypeOfHttpRequest.FORCE_ASYNC_ONLY_HTTP_REQUESTS) val clientConfig: AsyncHttpClientConfig = new AsyncHttpClientConfig.Builder() .setMaxConnections(5) .setUseProxyProperties(true) @@ -189,7 +190,6 @@ abstract class BaseOAuth2Authenticator extends OAuth2Authenticator { service } - } object BaseOAuth2Authenticator { @@ -200,18 +200,21 @@ object BaseOAuth2Authenticator { } def getAuthorizationUrl(config: OAuthConfig): String = { - val sb: StringBuilder = new StringBuilder(String.format(authorizeUrl, config.getResponseType, config.getApiKey, OAuthEncoder.encode(config.getCallback), OAuthEncoder.encode(config.getScope))) + val sb: StringBuilder = new StringBuilder(String.format(authorizeUrl, + config.getResponseType, config.getApiKey, OAuthEncoder.encode(config.getCallback), + OAuthEncoder.encode(config.getScope))) val state: String = config.getState if (state != null) { sb.append('&').append(OAuthConstants.STATE).append('=').append(OAuthEncoder.encode(state)) } - return sb.toString + sb.toString } override def createService(config: OAuthConfig): OAuth20Service = { - return new OAuth20Service(this, config) { + new OAuth20Service(this, config) { - protected override def createAccessTokenRequest[T <: AbstractRequest](code: String, request: T): T = { + protected override def createAccessTokenRequest[T <: AbstractRequest]( + code: String, request: T): T = { super.createAccessTokenRequest(code, request) if (!getConfig.hasGrantType) { diff --git a/services/jvm/src/main/scala/io/gearpump/services/security/oauth2/impl/CloudFoundryUAAOAuth2Authenticator.scala b/services/jvm/src/main/scala/io/gearpump/services/security/oauth2/impl/CloudFoundryUAAOAuth2Authenticator.scala index 27b66d227..98e37ea5c 100644 --- a/services/jvm/src/main/scala/io/gearpump/services/security/oauth2/impl/CloudFoundryUAAOAuth2Authenticator.scala +++ b/services/jvm/src/main/scala/io/gearpump/services/security/oauth2/impl/CloudFoundryUAAOAuth2Authenticator.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,44 +18,49 @@ package io.gearpump.services.security.oauth2.impl +import scala.concurrent.{ExecutionContext, Future, Promise} + +import com.typesafe.config.Config import com.github.scribejava.core.builder.api.DefaultApi20 import com.github.scribejava.core.model._ import com.github.scribejava.core.oauth.OAuth20Service import com.ning.http.client import com.ning.http.client.{AsyncCompletionHandler, AsyncHttpClient} -import com.typesafe.config.Config -import io.gearpump.services.SecurityService.UserSession -import io.gearpump.services.security.oauth2.OAuth2Authenticator -import io.gearpump.services.security.oauth2.impl.BaseOAuth2Authenticator.BaseApi20 -import io.gearpump.util.Constants._ import spray.json.{JsString, _} import sun.misc.BASE64Encoder -import scala.concurrent.{ExecutionContext, Promise, Future} +import io.gearpump.services.SecurityService.UserSession +import io.gearpump.services.security.oauth2.impl.BaseOAuth2Authenticator.BaseApi20 +import io.gearpump.util.Constants._ /** * * Does authentication with CloudFoundry UAA service. Currently it only * extract the email address of end user. * - * For what is UAA, please see: - * @see [[https://github.com/cloudfoundry/uaa for information about CloudFoundry UAA]] + * For what is UAA, + * See [[https://github.com/cloudfoundry/uaa for information about CloudFoundry UAA]] * (User Account and Authentication Service) * * Pre-requisite steps to use this Authenticator: * * Step1: Register your website to UAA with tool uaac. - * 1) Check tutorial on uaac at [[https://docs.cloudfoundry.org/adminguide/uaa-user-management.html]] - * 2) Open a bash shell, set the UAA server by command `uaac target` - * {{{ - * uaac target [your uaa server url] - * }}} + * 1) Check tutorial on uaac at + * [[https://docs.cloudfoundry.org/adminguide/uaa-user-management.html]] + * + * 2) Open a bash shell, set the UAA server by command `uaac target` + * {{{ + * uaac target [your uaa server url] + * }}} + * * NOTE: [your uaa server url] should match the uaahost settings in gear.conf - * 3) Login in as user admin by - * {{{ - * uaac token client get admin -s MyAdminPassword - * }}} - * 4) Create a new Application (Client) in UAA, + * + * 3) Login in as user admin by + * {{{ + * uaac token client get admin -s MyAdminPassword + * }}} + * + * 4) Create a new Application (Client) in UAA, * {{{ * uaac client add [your_client_id] * --scope "openid cloud_controller.read" @@ -67,28 +72,34 @@ import scala.concurrent.{ExecutionContext, Promise, Future} * }}} * * Step2: Configure the OAuth2 information in gear.conf - * 1) Enable OAuth2 authentication by setting "gearpump.ui-security.oauth2-authenticator-enabled" + * + * 1) Enable OAuth2 authentication by setting "gearpump.ui-security.oauth2-authenticator-enabled" * as true. - * 2) Navigate to section "gearpump.ui-security.oauth2-authenticators.cloudfoundryuaa" - * 3) Config gear.conf "gearpump.ui-security.oauth2-authenticators.cloudfoundryuaa" section. + * + * 2) Navigate to section "gearpump.ui-security.oauth2-authenticators.cloudfoundryuaa" + * + * 3) Config gear.conf "gearpump.ui-security.oauth2-authenticators.cloudfoundryuaa" section. * Please make sure class name, client ID, client Secret, and callback URL are set properly. * - * @note The callback URL here should match what you set on CloudFoundry UAA in step1. + * NOTE: The callback URL here should match what you set on CloudFoundry UAA in step1. * * Step3: Restart the UI service and try the "social login" button for UAA. * - * @note OAuth requires Internet access, @see [[OAuth2Authenticator]] to find tutorials to configure - * Internet proxy. + * NOTE: OAuth requires Internet access, @see + * [[io.gearpump.services.security.oauth2.OAuth2Authenticator]] to find tutorials to + * configure Internet proxy. * - * @see [[OAuth2Authenticator]] for more background information of OAuth2. + * See [[io.gearpump.services.security.oauth2.OAuth2Authenticator]] for more background + * information of OAuth2. */ class CloudFoundryUAAOAuth2Authenticator extends BaseOAuth2Authenticator { - import CloudFoundryUAAOAuth2Authenticator._ + import io.gearpump.services.security.oauth2.impl.CloudFoundryUAAOAuth2Authenticator._ private var host: String = null - protected override def authorizeUrl: String = s"$host/oauth/authorize?response_type=%s&client_id=%s&redirect_uri=%s&scope=%s" + protected override def authorizeUrl: String = + s"$host/oauth/authorize?response_type=%s&client_id=%s&redirect_uri=%s&scope=%s" protected override def accessTokenEndpoint: String = s"$host/oauth/token" @@ -104,7 +115,8 @@ class CloudFoundryUAAOAuth2Authenticator extends BaseOAuth2Authenticator { if (config.getBoolean(ADDITIONAL_AUTHENTICATOR_ENABLED)) { val additionalAuthenticatorConfig = config.getConfig(ADDITIONAL_AUTHENTICATOR) - val authenticatorClass = additionalAuthenticatorConfig.getString(GEARPUMP_UI_OAUTH2_AUTHENTICATOR_CLASS) + val authenticatorClass = additionalAuthenticatorConfig + .getString(GEARPUMP_UI_OAUTH2_AUTHENTICATOR_CLASS) val clazz = Thread.currentThread().getContextClassLoader.loadClass(authenticatorClass) val authenticator = clazz.newInstance().asInstanceOf[AdditionalAuthenticator] authenticator.init(additionalAuthenticatorConfig, executionContext) @@ -117,17 +129,17 @@ class CloudFoundryUAAOAuth2Authenticator extends BaseOAuth2Authenticator { email.value } - protected override def oauth2Api(): DefaultApi20 = { new CloudFoundryUAAService(authorizeUrl, accessTokenEndpoint) } - protected override def authenticateWithAccessToken(accessToken: OAuth2AccessToken): Future[UserSession] = { + protected override def authenticateWithAccessToken(accessToken: OAuth2AccessToken) + : Future[UserSession] = { implicit val ec: ExecutionContext = executionContext if (additionalAuthenticator.isDefined) { - super.authenticateWithAccessToken(accessToken).flatMap{user => + super.authenticateWithAccessToken(accessToken).flatMap { user => additionalAuthenticator.get.authenticate(oauthService.getAsyncHttpClient, accessToken, user) } } else { @@ -152,9 +164,10 @@ object CloudFoundryUAAOAuth2Authenticator { } override def createService(config: OAuthConfig): OAuth20Service = { - return new OAuth20Service(this, config) { + new OAuth20Service(this, config) { - protected override def createAccessTokenRequest[T <: AbstractRequest](code: String, request: T): T = { + protected override def createAccessTokenRequest[T <: AbstractRequest]( + code: String, request: T): T = { val config: OAuthConfig = getConfig() request.addParameter(OAuthConstants.GRANT_TYPE, OAuthConstants.AUTHORIZATION_CODE) @@ -181,8 +194,10 @@ object CloudFoundryUAAOAuth2Authenticator { trait AdditionalAuthenticator { /** - * @param config configurations specifically used for this authenticator. - * @param executionContext execution Context to use to run futures. + * Initialization + * + * @param config Configurations specifically used for this authenticator. + * @param executionContext Execution Context to use to run futures. */ def init(config: Config, executionContext: ExecutionContext): Unit @@ -192,7 +207,9 @@ object CloudFoundryUAAOAuth2Authenticator { * @param user user session returned by previous authenticator * @return an updated UserSession */ - def authenticate(asyncClient: AsyncHttpClient, accessToken: OAuth2AccessToken, user: UserSession): Future[UserSession] + def authenticate( + asyncClient: AsyncHttpClient, accessToken: OAuth2AccessToken, user: UserSession) + : Future[UserSession] } val ORGANIZATION_URL = "organization-url" diff --git a/services/jvm/src/main/scala/io/gearpump/services/security/oauth2/impl/GoogleOAuth2Authenticator.scala b/services/jvm/src/main/scala/io/gearpump/services/security/oauth2/impl/GoogleOAuth2Authenticator.scala index 54e6eef2b..c2b18fd46 100644 --- a/services/jvm/src/main/scala/io/gearpump/services/security/oauth2/impl/GoogleOAuth2Authenticator.scala +++ b/services/jvm/src/main/scala/io/gearpump/services/security/oauth2/impl/GoogleOAuth2Authenticator.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -22,8 +22,6 @@ import com.github.scribejava.apis.google.GoogleJsonTokenExtractor import com.github.scribejava.core.builder.api.DefaultApi20 import com.github.scribejava.core.extractors.TokenExtractor import com.github.scribejava.core.model._ -import io.gearpump.services.security.oauth2.OAuth2Authenticator -import io.gearpump.services.security.oauth2.impl.GoogleOAuth2Authenticator.AsyncGoogleApi20 import spray.json._ /** @@ -34,34 +32,40 @@ import spray.json._ * Pre-requisite steps to use this Authenticator: * * Step1: Register your website as an OAuth2 Application on Google - * 1) Create an application representing your website at [[https://console.developers.google.com]] - * 2) In "API Manager" of your created application, enable API "Google+ API" - * 3) Create OAuth client ID for this application. In "Credentials" tab of "API Manager", + * 1) Create an application representing your website at [[https://console.developers.google.com]] + * + * 2) In "API Manager" of your created application, enable API "Google+ API" + * + * 3) Create OAuth client ID for this application. In "Credentials" tab of "API Manager", * choose "Create credentials", and then select OAuth client ID. Follow the wizard * to set callback URL, and generate client ID, and client Secret. Callback URL is NOT optional. * * Step2: Configure the OAuth2 information in gear.conf - * 1) Enable OAuth2 authentication by setting "gearpump.ui-security.oauth2-authenticator-enabled" + * + * 1) Enable OAuth2 authentication by setting "gearpump.ui-security.oauth2-authenticator-enabled" * as true. - * 2) Configure section "gearpump.ui-security.oauth2-authenticators.google". Please make sure + * + * 2) Configure section "gearpump.ui-security.oauth2-authenticators.google". Please make sure * class name, client ID, client Secret, and callback URL are set properly. * - * @note callback URL set here should match what is configured on Google in step1. + * NOTE: callback URL set here should match what is configured on Google in step1. * * Step3: Restart the UI service and try out the Google social login button in UI. * - * @note OAuth requires Internet access, @see [[OAuth2Authenticator]] to find some helpful tutorials. + * NOTE: OAuth requires Internet access, @see + * [[io.gearpump.services.security.oauth2.OAuth2Authenticator]] to find some helpful tutorials * - * @note Google use scope to define what data can be fetched by OAuth2. Currently we use profile - * [[https://www.googleapis.com/auth/userinfo.email]]. However, Google may change the profile in future. + * NOTE: Google use scope to define what data can be fetched by OAuth2. Currently we use profile + * [[https://www.googleapis.com/auth/userinfo.email]]. However, Google may change the profile + * in future. * - * @todo Currently, this doesn't verify the state from Google OAuth2 response. + * TODO Currently, this doesn't verify the state from Google OAuth2 response. * - * @see [[OAuth2Authenticator]] for more API information. + * See [[io.gearpump.services.security.oauth2.OAuth2Authenticator]] for more API information. */ class GoogleOAuth2Authenticator extends BaseOAuth2Authenticator { - import GoogleOAuth2Authenticator._ + import io.gearpump.services.security.oauth2.impl.GoogleOAuth2Authenticator._ protected override def authorizeUrl: String = AuthorizeUrl @@ -83,15 +87,17 @@ class GoogleOAuth2Authenticator extends BaseOAuth2Authenticator { object GoogleOAuth2Authenticator { - import BaseOAuth2Authenticator._ + import io.gearpump.services.security.oauth2.impl.BaseOAuth2Authenticator._ + // scalastyle:off line.size.limit val AuthorizeUrl = "https://accounts.google.com/o/oauth2/auth?response_type=%s&client_id=%s&redirect_uri=%s&scope=%s" + // scalastyle:on line.size.limit val AccessEndpoint = "https://www.googleapis.com/oauth2/v4/token" val ResourceUrl = "https://www.googleapis.com/plus/v1/people/me" val Scope = "https://www.googleapis.com/auth/userinfo.email" private class AsyncGoogleApi20(authorizeUrl: String, accessEndpoint: String) - extends BaseApi20(authorizeUrl, accessEndpoint) { + extends BaseApi20(authorizeUrl, accessEndpoint) { override def getAccessTokenExtractor: TokenExtractor[OAuth2AccessToken] = { GoogleJsonTokenExtractor.instance diff --git a/services/jvm/src/main/scala/io/gearpump/services/util/UpickleUtil.scala b/services/jvm/src/main/scala/io/gearpump/services/util/UpickleUtil.scala index 0ae450581..f95474e45 100644 --- a/services/jvm/src/main/scala/io/gearpump/services/util/UpickleUtil.scala +++ b/services/jvm/src/main/scala/io/gearpump/services/util/UpickleUtil.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,19 +18,23 @@ package io.gearpump.services.util -import io.gearpump.WorkerId -import io.gearpump.util.Graph import upickle.Js +import io.gearpump.cluster.worker.WorkerId +import io.gearpump.util.Graph + object UpickleUtil { - //TODO: upickle cannot infer the reader automatically due to - // issue https://github.com/lihaoyi/upickle-pprint/issues/102 - implicit val graphReader: upickle.default.Reader[Graph[Int, String]] = upickle.default.Reader[Graph[Int, String]] { - case Js.Obj(verties, edges) => - val vertexList = upickle.default.readJs[List[Int]](verties._2) - val edgeList = upickle.default.readJs[List[(Int, String, Int)]](edges._2) - Graph(vertexList, edgeList) + // For implicit type, we need to add EXPLICIT return type, otherwise, upickle may NOT infer the + // reader type automatically. + // See issue https://github.com/lihaoyi/upickle-pprint/issues/102 + implicit val graphReader: upickle.default.Reader[Graph[Int, String]] = { + upickle.default.Reader[Graph[Int, String]] { + case Js.Obj(verties, edges) => + val vertexList = upickle.default.readJs[List[Int]](verties._2) + val edgeList = upickle.default.readJs[List[(Int, String, Int)]](edges._2) + Graph(vertexList, edgeList) + } } implicit val workerIdReader: upickle.default.Reader[WorkerId] = upickle.default.Reader[WorkerId] { diff --git a/services/jvm/src/test/scala/io/gearpump/services/AdminServiceSpec.scala b/services/jvm/src/test/scala/io/gearpump/services/AdminServiceSpec.scala index 738bbad68..e67731e72 100644 --- a/services/jvm/src/test/scala/io/gearpump/services/AdminServiceSpec.scala +++ b/services/jvm/src/test/scala/io/gearpump/services/AdminServiceSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,17 +18,21 @@ package io.gearpump.services +import scala.concurrent.Await +import scala.concurrent.duration._ + import akka.actor.ActorSystem +import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} +import com.typesafe.config.Config +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} + import io.gearpump.cluster.TestUtil -import akka.http.scaladsl.testkit.{ScalatestRouteTest, RouteTestTimeout} -import com.typesafe.config.{Config, ConfigFactory} -import org.scalatest.{BeforeAndAfterAll, Matchers, FlatSpec} -import io.gearpump.services.util.UpickleUtil._ -import scala.concurrent.duration._ -import scala.util.Try +// NOTE: This cannot be removed!!! +import io.gearpump.services.util.UpickleUtil._ -class AdminServiceSpec extends FlatSpec with ScalatestRouteTest with Matchers with BeforeAndAfterAll { +class AdminServiceSpec + extends FlatSpec with ScalatestRouteTest with Matchers with BeforeAndAfterAll { override def testConfig: Config = TestUtil.DEFAULT_CONFIG @@ -36,14 +40,14 @@ class AdminServiceSpec extends FlatSpec with ScalatestRouteTest with Matchers w it should "shutdown the ActorSystem when receiving terminate" in { val route = new AdminService(actorSystem).route - implicit val customTimeout = RouteTestTimeout(15 seconds) - (Post(s"/terminate") ~> route) ~> check{ + implicit val customTimeout = RouteTestTimeout(15.seconds) + (Post(s"/terminate") ~> route) ~> check { assert(status.intValue() == 404) } - actorSystem.awaitTermination(20 seconds) + Await.result(actorSystem.whenTerminated, 20.seconds) // terminate should terminate current actor system - assert(actorSystem.isTerminated) + assert(actorSystem.whenTerminated.isCompleted) } } diff --git a/services/jvm/src/test/scala/io/gearpump/services/AppMasterServiceSpec.scala b/services/jvm/src/test/scala/io/gearpump/services/AppMasterServiceSpec.scala index 7b98cb6ae..59da80d49 100644 --- a/services/jvm/src/test/scala/io/gearpump/services/AppMasterServiceSpec.scala +++ b/services/jvm/src/test/scala/io/gearpump/services/AppMasterServiceSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,13 +18,19 @@ package io.gearpump.services +import scala.concurrent.duration._ +import scala.util.{Success, Try} + import akka.actor.ActorRef import akka.http.scaladsl.model.headers.`Cache-Control` -import akka.http.scaladsl.server.RouteResult +import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} import akka.testkit.TestActor.{AutoPilot, KeepRunning} import akka.testkit.{TestKit, TestProbe} import com.typesafe.config.{Config, ConfigFactory} -import akka.http.scaladsl.testkit.RouteTestTimeout +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} +import org.slf4j.Logger +import upickle.default.read + import io.gearpump.cluster.AppMasterToMaster.GeneralAppMasterSummary import io.gearpump.cluster.ClientToMaster.{GetLastFailure, QueryAppMasterConfig, QueryHistoryMetrics, ResolveAppId} import io.gearpump.cluster.MasterToAppMaster.{AppMasterData, AppMasterDataDetailRequest, AppMasterDataRequest} @@ -33,13 +39,8 @@ import io.gearpump.cluster.TestUtil import io.gearpump.jarstore.JarStoreService import io.gearpump.streaming.executor.Executor.{ExecutorConfig, ExecutorSummary, GetExecutorSummary, QueryExecutorConfig} import io.gearpump.util.LogUtil -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import org.slf4j.Logger -import upickle.default.read -import akka.http.scaladsl.testkit.ScalatestRouteTest +// NOTE: This cannot be removed!!! import io.gearpump.services.util.UpickleUtil._ -import scala.concurrent.duration._ -import scala.util.{Success, Try} class AppMasterServiceSpec extends FlatSpec with ScalatestRouteTest with Matchers with BeforeAndAfterAll { @@ -47,7 +48,7 @@ class AppMasterServiceSpec extends FlatSpec with ScalatestRouteTest override def testConfig: Config = TestUtil.UI_CONFIG private val LOG: Logger = LogUtil.getLogger(getClass) - def actorRefFactory = system + private def actorRefFactory = system val mockAppMaster = TestProbe() val failure = LastFailure(System.currentTimeMillis(), "Some error") @@ -56,13 +57,13 @@ class AppMasterServiceSpec extends FlatSpec with ScalatestRouteTest def jarStore: JarStoreService = jarStoreService - def master = mockMaster.ref + private def master = mockMaster.ref - def appMasterRoute = new AppMasterService(master, jarStore, system).route + private def appMasterRoute = new AppMasterService(master, jarStore, system).route mockAppMaster.setAutoPilot { new AutoPilot { - def run(sender: ActorRef, msg: Any) = msg match { + def run(sender: ActorRef, msg: Any): AutoPilot = msg match { case AppMasterDataDetailRequest(appId) => sender ! GeneralAppMasterSummary(appId) KeepRunning @@ -78,7 +79,6 @@ class AppMasterServiceSpec extends FlatSpec with ScalatestRouteTest case QueryExecutorConfig(0) => sender ! ExecutorConfig(system.settings.config) KeepRunning - } } } @@ -86,7 +86,7 @@ class AppMasterServiceSpec extends FlatSpec with ScalatestRouteTest val mockMaster = TestProbe() mockMaster.setAutoPilot { new AutoPilot { - def run(sender: ActorRef, msg: Any) = msg match { + def run(sender: ActorRef, msg: Any): AutoPilot = msg match { case ResolveAppId(0) => sender ! ResolveAppIdResult(Success(mockAppMaster.ref)) KeepRunning @@ -102,20 +102,19 @@ class AppMasterServiceSpec extends FlatSpec with ScalatestRouteTest "AppMasterService" should "return a JSON structure for GET request when detail = false" in { implicit val customTimeout = RouteTestTimeout(15.seconds) - Get(s"/api/$REST_VERSION/appmaster/0?detail=false") ~> appMasterRoute ~> check{ + Get(s"/api/$REST_VERSION/appmaster/0?detail=false") ~> appMasterRoute ~> check { val responseBody = responseAs[String] read[AppMasterData](responseBody) - // check the header, should contains no-cache header. + // Checks the header, should contains no-cache header. // Cache-Control:no-cache, max-age=0 val noCache = header[`Cache-Control`].get.value() assert(noCache == "no-cache, max-age=0") } - Get(s"/api/$REST_VERSION/appmaster/0?detail=true") ~> appMasterRoute ~> check{ + Get(s"/api/$REST_VERSION/appmaster/0?detail=true") ~> appMasterRoute ~> check { val responseBody = responseAs[String] } - } "MetricsQueryService" should "return history metrics" in { @@ -137,7 +136,7 @@ class AppMasterServiceSpec extends FlatSpec with ScalatestRouteTest "ConfigQueryService" should "return config for application" in { implicit val customTimeout = RouteTestTimeout(15.seconds) - (Get(s"/api/$REST_VERSION/appmaster/0/config") ~> appMasterRoute) ~> check{ + (Get(s"/api/$REST_VERSION/appmaster/0/config") ~> appMasterRoute) ~> check { val responseBody = responseAs[String] val config = Try(ConfigFactory.parseString(responseBody)) assert(config.isSuccess) @@ -146,7 +145,7 @@ class AppMasterServiceSpec extends FlatSpec with ScalatestRouteTest it should "return config for executor " in { implicit val customTimeout = RouteTestTimeout(15.seconds) - (Get(s"/api/$REST_VERSION/appmaster/0/executor/0/config") ~> appMasterRoute) ~> check{ + (Get(s"/api/$REST_VERSION/appmaster/0/executor/0/config") ~> appMasterRoute) ~> check { val responseBody = responseAs[String] val config = Try(ConfigFactory.parseString(responseBody)) assert(config.isSuccess) @@ -155,14 +154,13 @@ class AppMasterServiceSpec extends FlatSpec with ScalatestRouteTest it should "return return executor summary" in { implicit val customTimeout = RouteTestTimeout(15.seconds) - (Get(s"/api/$REST_VERSION/appmaster/0/executor/0") ~> appMasterRoute) ~> check{ + (Get(s"/api/$REST_VERSION/appmaster/0/executor/0") ~> appMasterRoute) ~> check { val responseBody = responseAs[String] val executorSummary = read[ExecutorSummary](responseBody) assert(executorSummary.id == 0) } } - override def afterAll { TestKit.shutdownActorSystem(system) } diff --git a/services/jvm/src/test/scala/io/gearpump/services/MasterServiceSpec.scala b/services/jvm/src/test/scala/io/gearpump/services/MasterServiceSpec.scala index 1cfc01a0b..a24db189f 100644 --- a/services/jvm/src/test/scala/io/gearpump/services/MasterServiceSpec.scala +++ b/services/jvm/src/test/scala/io/gearpump/services/MasterServiceSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,55 +19,55 @@ package io.gearpump.services import java.io.File +import scala.concurrent.duration._ +import scala.concurrent.{ExecutionContext, Future} +import scala.util.{Success, Try} import akka.actor.ActorRef import akka.http.scaladsl.marshalling.Marshal import akka.http.scaladsl.model._ -import akka.http.scaladsl.model.headers.{`Cache-Control`, `Set-Cookie`} -import akka.stream.scaladsl.Source +import akka.http.scaladsl.model.headers.`Cache-Control` +import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} +import akka.stream.scaladsl.{FileIO, Source} import akka.testkit.TestActor.{AutoPilot, KeepRunning} import akka.testkit.TestProbe import com.typesafe.config.{Config, ConfigFactory} -import io.gearpump.WorkerId +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} + import io.gearpump.cluster.AppMasterToMaster.{GetAllWorkers, GetMasterData, GetWorkerData, MasterData, WorkerData} import io.gearpump.cluster.ClientToMaster.{QueryHistoryMetrics, QueryMasterConfig, ResolveWorkerId, SubmitApplication} import io.gearpump.cluster.MasterToAppMaster.{AppMasterData, AppMastersData, AppMastersDataRequest, WorkerList} import io.gearpump.cluster.MasterToClient._ import io.gearpump.cluster.TestUtil -import io.gearpump.cluster.worker.WorkerSummary -import io.gearpump.services.MasterService.{SubmitApplicationRequest, BuiltinPartitioners} +import io.gearpump.cluster.worker.{WorkerId, WorkerSummary} import io.gearpump.jarstore.JarStoreService -import io.gearpump.streaming.ProcessorDescription -import io.gearpump.util.{Constants, Graph} -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} -import scala.concurrent.{Future, ExecutionContext} -import scala.concurrent.duration._ -import scala.util.{Success, Try} -import akka.stream.scaladsl.FileIO +import io.gearpump.services.MasterService.{BuiltinPartitioners, SubmitApplicationRequest} +// NOTE: This cannot be removed!!! import io.gearpump.services.util.UpickleUtil._ +import io.gearpump.streaming.ProcessorDescription +import io.gearpump.util.Graph -class MasterServiceSpec extends FlatSpec with ScalatestRouteTest with - Matchers with BeforeAndAfterAll { +class MasterServiceSpec extends FlatSpec with ScalatestRouteTest + with Matchers with BeforeAndAfterAll { import upickle.default.{read, write} override def testConfig: Config = TestUtil.UI_CONFIG - def actorRefFactory = system + private def actorRefFactory = system val workerId = 0 val mockWorker = TestProbe() lazy val jarStoreService = JarStoreService.get(system.settings.config) - def master = mockMaster.ref + private def master = mockMaster.ref def jarStore: JarStoreService = jarStoreService - def masterRoute = new MasterService(master, jarStore, system).route + private def masterRoute = new MasterService(master, jarStore, system).route mockWorker.setAutoPilot { new AutoPilot { - def run(sender: ActorRef, msg: Any) = msg match { + def run(sender: ActorRef, msg: Any): AutoPilot = msg match { case GetWorkerData(workerId) => sender ! WorkerData(WorkerSummary.empty.copy(state = "active", workerId = workerId)) KeepRunning @@ -81,11 +81,11 @@ class MasterServiceSpec extends FlatSpec with ScalatestRouteTest with val mockMaster = TestProbe() mockMaster.setAutoPilot { new AutoPilot { - def run(sender: ActorRef, msg: Any) = msg match { + def run(sender: ActorRef, msg: Any): AutoPilot = msg match { case GetMasterData => sender ! MasterData(null) KeepRunning - case AppMastersDataRequest => + case AppMastersDataRequest => sender ! AppMastersData(List.empty[AppMasterData]) KeepRunning case GetAllWorkers => @@ -107,15 +107,14 @@ class MasterServiceSpec extends FlatSpec with ScalatestRouteTest with } } - it should "return master info when asked" in { implicit val customTimeout = RouteTestTimeout(15.seconds) (Get(s"/api/$REST_VERSION/master") ~> masterRoute) ~> check { - // check the type + // Checks the type val content = responseAs[String] read[MasterData](content) - // check the header, should contains no-cache header. + // Checks the header, should contains no-cache header. // Cache-Control:no-cache, max-age=0 val noCache = header[`Cache-Control`].get.value() assert(noCache == "no-cache, max-age=0") @@ -127,7 +126,7 @@ class MasterServiceSpec extends FlatSpec with ScalatestRouteTest with it should "return a json structure of appMastersData for GET request" in { implicit val customTimeout = RouteTestTimeout(15.seconds) (Get(s"/api/$REST_VERSION/master/applist") ~> masterRoute) ~> check { - //check the type + // Checks the type read[AppMastersData](responseAs[String]) } mockMaster.expectMsg(AppMastersDataRequest) @@ -136,7 +135,7 @@ class MasterServiceSpec extends FlatSpec with ScalatestRouteTest with it should "return a json structure of worker data for GET request" in { implicit val customTimeout = RouteTestTimeout(25.seconds) Get(s"/api/$REST_VERSION/master/workerlist") ~> masterRoute ~> check { - //check the type + // Checks the type val workerListJson = responseAs[String] val workers = read[List[WorkerSummary]](workerListJson) assert(workers.size > 0) @@ -151,7 +150,7 @@ class MasterServiceSpec extends FlatSpec with ScalatestRouteTest with it should "return config for master" in { implicit val customTimeout = RouteTestTimeout(15.seconds) - (Get(s"/api/$REST_VERSION/master/config") ~> masterRoute) ~> check{ + (Get(s"/api/$REST_VERSION/master/config") ~> masterRoute) ~> check { val responseBody = responseAs[String] val config = Try(ConfigFactory.parseString(responseBody)) assert(config.isSuccess) @@ -170,7 +169,7 @@ class MasterServiceSpec extends FlatSpec with ScalatestRouteTest with } private def entity(file: File)(implicit ec: ExecutionContext): Future[RequestEntity] = { - val entity = HttpEntity(MediaTypes.`application/octet-stream`, file.length(), + val entity = HttpEntity(MediaTypes.`application/octet-stream`, file.length(), FileIO.fromFile(file, chunkSize = 100000)) val body = Source.single( @@ -185,7 +184,7 @@ class MasterServiceSpec extends FlatSpec with ScalatestRouteTest with "MetricsQueryService" should "return history metrics" in { implicit val customTimeout = RouteTestTimeout(15.seconds) - (Get(s"/api/$REST_VERSION/master/metrics/master") ~> masterRoute)~> check { + (Get(s"/api/$REST_VERSION/master/metrics/master") ~> masterRoute) ~> check { val responseBody = responseAs[String] val config = Try(ConfigFactory.parseString(responseBody)) assert(config.isSuccess) @@ -200,7 +199,8 @@ class MasterServiceSpec extends FlatSpec with ScalatestRouteTest with ) val dag = Graph(0 ~ "partitioner" ~> 1) val jsonValue = write(SubmitApplicationRequest("complexdag", processors, dag, null)) - Post(s"/api/$REST_VERSION/master/submitdag", HttpEntity(ContentTypes.`application/json`, jsonValue)) ~> masterRoute ~> check { + Post(s"/api/$REST_VERSION/master/submitdag", + HttpEntity(ContentTypes.`application/json`, jsonValue)) ~> masterRoute ~> check { val responseBody = responseAs[String] val submitApplicationResultValue = read[SubmitApplicationResultValue](responseBody) assert(submitApplicationResultValue.appId >= 0, "invalid appid") diff --git a/services/jvm/src/test/scala/io/gearpump/services/SecurityServiceSpec.scala b/services/jvm/src/test/scala/io/gearpump/services/SecurityServiceSpec.scala index 10b45b928..3cf99e9b1 100644 --- a/services/jvm/src/test/scala/io/gearpump/services/SecurityServiceSpec.scala +++ b/services/jvm/src/test/scala/io/gearpump/services/SecurityServiceSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,23 +18,23 @@ package io.gearpump.services -import akka.http.scaladsl.testkit.{RouteTestTimeout} -import com.typesafe.config.Config -import io.gearpump.cluster.TestUtil -import io.gearpump.services.util.UpickleUtil._ -import org.scalatest.{BeforeAndAfterAll, Matchers, FlatSpec} -import akka.actor.{ActorSystem} -import akka.http.scaladsl.server._ import scala.concurrent.duration._ -import akka.http.scaladsl.model._ -import headers._ + +import akka.actor.ActorSystem import akka.http.scaladsl.model.FormData -import akka.http.scaladsl.model.headers.{Cookie, `Set-Cookie`} -import akka.http.scaladsl.server.AuthorizationFailedRejection +import akka.http.scaladsl.model.headers.{Cookie, `Set-Cookie`, _} import akka.http.scaladsl.server.Directives._ -import akka.http.scaladsl.testkit.ScalatestRouteTest +import akka.http.scaladsl.server.{AuthorizationFailedRejection, _} +import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} +import com.typesafe.config.Config +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -class SecurityServiceSpec extends FlatSpec with ScalatestRouteTest with Matchers with BeforeAndAfterAll { +import io.gearpump.cluster.TestUtil +// NOTE: This cannot be removed!!! +import io.gearpump.services.util.UpickleUtil._ + +class SecurityServiceSpec + extends FlatSpec with ScalatestRouteTest with Matchers with BeforeAndAfterAll { override def testConfig: Config = TestUtil.UI_CONFIG @@ -43,9 +43,9 @@ class SecurityServiceSpec extends FlatSpec with ScalatestRouteTest with Matcher it should "return 401 if not authenticated" in { val security = new SecurityService(SecurityServiceSpec.resource, actorSystem) - implicit val customTimeout = RouteTestTimeout(15 seconds) + implicit val customTimeout = RouteTestTimeout(15.seconds) - (Get(s"/resource") ~> security.route) ~> check{ + (Get(s"/resource") ~> security.route) ~> check { assert(rejection.isInstanceOf[AuthenticationFailedRejection]) } } @@ -53,10 +53,11 @@ class SecurityServiceSpec extends FlatSpec with ScalatestRouteTest with Matcher "guest" should "get protected resource after authentication" in { val security = new SecurityService(SecurityServiceSpec.resource, actorSystem) - implicit val customTimeout = RouteTestTimeout(15 seconds) + implicit val customTimeout = RouteTestTimeout(15.seconds) var cookie: HttpCookiePair = null - (Post(s"/login", FormData("username" -> "guest", "password" -> "guest")) ~> security.route) ~> check{ + (Post(s"/login", FormData("username" -> "guest", "password" -> "guest")) + ~> security.route) ~> check { assert("{\"user\":\"guest\"}" == responseAs[String]) assert(status.intValue() == 200) assert(header[`Set-Cookie`].isDefined) @@ -65,18 +66,18 @@ class SecurityServiceSpec extends FlatSpec with ScalatestRouteTest with Matcher cookie = HttpCookiePair.apply(httpCookie.name, httpCookie.value) } - // after authentication, everything is fine. + // After authentication, everything is fine. Get("/resource").addHeader(Cookie(cookie)) ~> security.route ~> check { responseAs[String] shouldEqual "OK" } - // however, guest cannot access high-permission operations, like POST. - Post("/resource").addHeader(Cookie(cookie)) ~> security.route ~> check { + // However, guest cannot access high-permission operations, like POST. + Post("/resource").addHeader(Cookie(cookie)) ~> security.route ~> check { assert(rejection == AuthorizationFailedRejection) } - // logout, should clear the session - Post(s"/logout").addHeader(Cookie(cookie)) ~> security.route ~> check{ + // Logout, should clear the session + Post(s"/logout").addHeader(Cookie(cookie)) ~> security.route ~> check { assert("{\"user\":\"guest\"}" == responseAs[String]) assert(status.intValue() == 200) assert(header[`Set-Cookie`].isDefined) @@ -85,12 +86,12 @@ class SecurityServiceSpec extends FlatSpec with ScalatestRouteTest with Matcher assert(httpCookie.value == "deleted") } - // access again, rejected. - Get("/resource") ~> security.route ~> check { + // Access again, rejected this time. + Get("/resource") ~> security.route ~> check { assert(rejection.isInstanceOf[AuthenticationFailedRejection]) } - Post("/resource") ~> security.route ~> check { + Post("/resource") ~> security.route ~> check { assert(rejection.isInstanceOf[AuthenticationFailedRejection]) } } @@ -98,10 +99,11 @@ class SecurityServiceSpec extends FlatSpec with ScalatestRouteTest with Matcher "admin" should "get protected resource after authentication" in { val security = new SecurityService(SecurityServiceSpec.resource, actorSystem) - implicit val customTimeout = RouteTestTimeout(15 seconds) + implicit val customTimeout = RouteTestTimeout(15.seconds) var cookie: HttpCookiePair = null - (Post(s"/login", FormData("username" -> "admin", "password" -> "admin")) ~> security.route) ~> check{ + (Post(s"/login", FormData("username" -> "admin", "password" -> "admin")) + ~> security.route) ~> check { assert("{\"user\":\"admin\"}" == responseAs[String]) assert(status.intValue() == 200) assert(header[`Set-Cookie`].isDefined) @@ -110,7 +112,7 @@ class SecurityServiceSpec extends FlatSpec with ScalatestRouteTest with Matcher cookie = HttpCookiePair(httpCookie.name, httpCookie.value) } - // after authentication, everything is fine. + // After authentication, everything is fine. Get("/resource").addHeader(Cookie(cookie)) ~> security.route ~> check { responseAs[String] shouldEqual "OK" } @@ -120,8 +122,8 @@ class SecurityServiceSpec extends FlatSpec with ScalatestRouteTest with Matcher responseAs[String] shouldEqual "OK" } - // logout, should clear the session - Post(s"/logout").addHeader(Cookie(cookie)) ~> security.route ~> check{ + // Logout, should clear the session + Post(s"/logout").addHeader(Cookie(cookie)) ~> security.route ~> check { assert("{\"user\":\"admin\"}" == responseAs[String]) assert(status.intValue() == 200) assert(header[`Set-Cookie`].isDefined) @@ -130,8 +132,8 @@ class SecurityServiceSpec extends FlatSpec with ScalatestRouteTest with Matcher assert(httpCookie.value == "deleted") } - // access again, rejected. - Get("/resource") ~> security.route ~> check { + // Access again, rejected this time. + Get("/resource") ~> security.route ~> check { assert(rejection.isInstanceOf[AuthenticationFailedRejection]) } @@ -145,7 +147,7 @@ object SecurityServiceSpec { val resource = new RouteService { override def route: Route = { - get{ + get { path("resource") { complete("OK") } diff --git a/services/jvm/src/test/scala/io/gearpump/services/StaticServiceSpec.scala b/services/jvm/src/test/scala/io/gearpump/services/StaticServiceSpec.scala index 501776ebc..f61e2f5cb 100644 --- a/services/jvm/src/test/scala/io/gearpump/services/StaticServiceSpec.scala +++ b/services/jvm/src/test/scala/io/gearpump/services/StaticServiceSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,41 +18,43 @@ package io.gearpump.services +import scala.concurrent.duration._ +import scala.util.Try + import akka.http.scaladsl.model.headers.`Cache-Control` import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} -import akka.testkit.TestProbe import com.typesafe.config.{Config, ConfigFactory} +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} + import io.gearpump.cluster.TestUtil import io.gearpump.util.Constants +// NOTE: This cannot be removed!!! import io.gearpump.services.util.UpickleUtil._ -import org.scalatest.{BeforeAndAfterAll, Matchers, FlatSpec} -import scala.util.Try -import scala.concurrent.duration._ - -class StaticServiceSpec extends FlatSpec with ScalatestRouteTest with Matchers with BeforeAndAfterAll { +class StaticServiceSpec + extends FlatSpec with ScalatestRouteTest with Matchers with BeforeAndAfterAll { override def testConfig: Config = TestUtil.UI_CONFIG - private val supervisorPath = system.settings.config.getString(Constants.GEARPUMP_SERVICE_SUPERVISOR_PATH) - + private val supervisorPath = system.settings.config.getString( + Constants.GEARPUMP_SERVICE_SUPERVISOR_PATH) - def route = new StaticService(system, supervisorPath).route + protected def route = new StaticService(system, supervisorPath).route it should "return version" in { - implicit val customTimeout = RouteTestTimeout(15 seconds) - (Get(s"/version") ~> route) ~> check{ + implicit val customTimeout = RouteTestTimeout(15.seconds) + (Get(s"/version") ~> route) ~> check { val responseBody = responseAs[String] val config = Try(ConfigFactory.parseString(responseBody)) assert(responseBody == "Unknown-Version") - // by default, it will be cached. + // By default, it will be cached. assert(header[`Cache-Control`].isEmpty) } } it should "get correct supervisor path" in { - implicit val customTimeout = RouteTestTimeout(15 seconds) - (Get(s"/supervisor-actor-path") ~> route) ~> check{ + implicit val customTimeout = RouteTestTimeout(15.seconds) + (Get(s"/supervisor-actor-path") ~> route) ~> check { val responseBody = responseAs[String] val defaultSupervisorPath = "" assert(responseBody == defaultSupervisorPath) diff --git a/services/jvm/src/test/scala/io/gearpump/services/WorkerServiceSpec.scala b/services/jvm/src/test/scala/io/gearpump/services/WorkerServiceSpec.scala index dc8d5a79e..2676a1648 100644 --- a/services/jvm/src/test/scala/io/gearpump/services/WorkerServiceSpec.scala +++ b/services/jvm/src/test/scala/io/gearpump/services/WorkerServiceSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,42 +18,44 @@ package io.gearpump.services +import scala.concurrent.duration._ +import scala.util.{Success, Try} + import akka.actor.ActorRef import akka.http.scaladsl.model.headers.`Cache-Control` +import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} import akka.testkit.TestActor.{AutoPilot, KeepRunning} import akka.testkit.{TestKit, TestProbe} import com.typesafe.config.{Config, ConfigFactory} -import io.gearpump.WorkerId +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} + import io.gearpump.cluster.AppMasterToMaster.{GetWorkerData, WorkerData} import io.gearpump.cluster.ClientToMaster.{QueryHistoryMetrics, QueryWorkerConfig, ResolveWorkerId} import io.gearpump.cluster.MasterToClient.{HistoryMetrics, HistoryMetricsItem, ResolveWorkerIdResult, WorkerConfig} import io.gearpump.cluster.TestUtil -import io.gearpump.cluster.worker.WorkerSummary +import io.gearpump.cluster.worker.{WorkerId, WorkerSummary} import io.gearpump.jarstore.JarStoreService +// NOTE: This cannot be removed!!! import io.gearpump.services.util.UpickleUtil._ -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} - -import scala.concurrent.duration._ -import scala.util.{Success, Try} -import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} -class WorkerServiceSpec extends FlatSpec with ScalatestRouteTest with Matchers with BeforeAndAfterAll { +class WorkerServiceSpec + extends FlatSpec with ScalatestRouteTest with Matchers with BeforeAndAfterAll { override def testConfig: Config = TestUtil.DEFAULT_CONFIG - def actorRefFactory = system + protected def actorRefFactory = system val mockWorker = TestProbe() - def master = mockMaster.ref + protected def master = mockMaster.ref lazy val jarStoreService = JarStoreService.get(system.settings.config) - def workerRoute = new WorkerService(master, system).route + protected def workerRoute = new WorkerService(master, system).route mockWorker.setAutoPilot { new AutoPilot { - def run(sender: ActorRef, msg: Any) = msg match { + def run(sender: ActorRef, msg: Any): AutoPilot = msg match { case GetWorkerData(workerId) => sender ! WorkerData(WorkerSummary.empty) KeepRunning @@ -70,7 +72,7 @@ class WorkerServiceSpec extends FlatSpec with ScalatestRouteTest with Matchers val mockMaster = TestProbe() mockMaster.setAutoPilot { new AutoPilot { - def run(sender: ActorRef, msg: Any) = msg match { + def run(sender: ActorRef, msg: Any): AutoPilot = msg match { case ResolveWorkerId(workerId) => sender ! ResolveWorkerIdResult(Success(mockWorker.ref)) KeepRunning @@ -78,10 +80,10 @@ class WorkerServiceSpec extends FlatSpec with ScalatestRouteTest with Matchers } } - "ConfigQueryService" should "return config for worker" in { implicit val customTimeout = RouteTestTimeout(15.seconds) - (Get(s"/api/$REST_VERSION/worker/${WorkerId.render(WorkerId(0, 0L))}/config") ~> workerRoute) ~> check{ + (Get(s"/api/$REST_VERSION/worker/${WorkerId.render(WorkerId(0, 0L))}/config") + ~> workerRoute) ~> check { val responseBody = responseAs[String] val config = Try(ConfigFactory.parseString(responseBody)) assert(config.isSuccess) @@ -90,12 +92,13 @@ class WorkerServiceSpec extends FlatSpec with ScalatestRouteTest with Matchers it should "return WorkerData" in { implicit val customTimeout = RouteTestTimeout(15.seconds) - (Get(s"/api/$REST_VERSION/worker/${WorkerId.render(WorkerId(1, 0L))}") ~> workerRoute) ~> check{ + (Get(s"/api/$REST_VERSION/worker/${WorkerId.render(WorkerId(1, 0L))}") + ~> workerRoute) ~> check { val responseBody = responseAs[String] val config = Try(ConfigFactory.parseString(responseBody)) assert(config.isSuccess) - // check the header, should contains no-cache header. + // Check the header, should contains no-cache header. // Cache-Control:no-cache, max-age=0 val noCache = header[`Cache-Control`].get.value() assert(noCache == "no-cache, max-age=0") @@ -104,7 +107,8 @@ class WorkerServiceSpec extends FlatSpec with ScalatestRouteTest with Matchers "MetricsQueryService" should "return history metrics" in { implicit val customTimeout = RouteTestTimeout(15.seconds) - (Get(s"/api/$REST_VERSION/worker/${WorkerId.render(WorkerId(0, 0L))}/metrics/worker") ~> workerRoute) ~> check { + (Get(s"/api/$REST_VERSION/worker/${WorkerId.render(WorkerId(0, 0L))}/metrics/worker") + ~> workerRoute) ~> check { val responseBody = responseAs[String] val config = Try(ConfigFactory.parseString(responseBody)) assert(config.isSuccess) diff --git a/services/jvm/src/test/scala/io/gearpump/services/security/oauth2/CloudFoundryUAAOAuth2AuthenticatorSpec.scala b/services/jvm/src/test/scala/io/gearpump/services/security/oauth2/CloudFoundryUAAOAuth2AuthenticatorSpec.scala index 68a3506b3..136026a23 100644 --- a/services/jvm/src/test/scala/io/gearpump/services/security/oauth2/CloudFoundryUAAOAuth2AuthenticatorSpec.scala +++ b/services/jvm/src/test/scala/io/gearpump/services/security/oauth2/CloudFoundryUAAOAuth2AuthenticatorSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,19 +18,23 @@ package io.gearpump.services.security.oauth2 +import scala.collection.JavaConverters._ +import scala.concurrent.Await +import scala.concurrent.duration._ + import akka.actor.ActorSystem import akka.http.scaladsl.model.HttpEntity.Strict import akka.http.scaladsl.model.MediaTypes._ -import akka.http.scaladsl.model.Uri.{Query, Path} +import akka.http.scaladsl.model.Uri.{Path, Query} import akka.http.scaladsl.model._ import akka.http.scaladsl.testkit.ScalatestRouteTest import com.typesafe.config.ConfigFactory -import io.gearpump.security.Authenticator -import io.gearpump.services.security.oauth2.impl.{CloudFoundryUAAOAuth2Authenticator, GoogleOAuth2Authenticator} import org.scalatest.FlatSpec -import scala.concurrent.Await -import scala.concurrent.duration._ -import scala.collection.JavaConverters._ + +import io.gearpump.security.Authenticator +// NOTE: This cannot be removed!!! +import io.gearpump.services.util.UpickleUtil._ +import io.gearpump.services.security.oauth2.impl.CloudFoundryUAAOAuth2Authenticator class CloudFoundryUAAOAuth2AuthenticatorSpec extends FlatSpec with ScalatestRouteTest { @@ -71,8 +75,9 @@ class CloudFoundryUAAOAuth2AuthenticatorSpec extends FlatSpec with ScalatestRout val refreshToken = "eyJhbGciOiJSUzI1NiJ9.eyJqdGkiOiI2Nm" val mail = "test@gearpump.io" - def accessTokenEndpoint(request: HttpRequest) = { - assert(request.getHeader("Authorization").get.value() == "Basic Z2VhcnB1bXBfdGVzdDI6Z2VhcnB1bXBfdGVzdDI=") + def accessTokenEndpoint(request: HttpRequest): HttpResponse = { + assert(request.getHeader("Authorization").get.value() == + "Basic Z2VhcnB1bXBfdGVzdDI6Z2VhcnB1bXBfdGVzdDI=") assert(request.entity.contentType.mediaType.value == "application/x-www-form-urlencoded") val body = request.entity.asInstanceOf[Strict].data.decodeString("UTF-8") @@ -85,27 +90,27 @@ class CloudFoundryUAAOAuth2AuthenticatorSpec extends FlatSpec with ScalatestRout val response = s""" - |{ - | "access_token": "$accessToken", - | "token_type": "bearer", - | "refresh_token": "$refreshToken", - | "expires_in": 43199, - | "scope": "openid", - | "jti": "e8739474-b2fa-42eb-a9ad-e065bf79d7e9" - |} + |{ + | "access_token": "$accessToken", + | "token_type": "bearer", + | "refresh_token": "$refreshToken", + | "expires_in": 43199, + | "scope": "openid", + | "jti": "e8739474-b2fa-42eb-a9ad-e065bf79d7e9" + |} """.stripMargin HttpResponse(entity = HttpEntity(ContentType(`application/json`), response)) } - def protectedResourceEndpoint(request: HttpRequest) = { + def protectedResourceEndpoint(request: HttpRequest): HttpResponse = { assert(request.getUri().query().get("access_token").get == accessToken) val response = s""" - |{ - | "user_id": "e2922002-0218-4513-a62d-1da2ba64ee4c", - | "user_name": "user", - | "email": "$mail" - |} + |{ + | "user_id": "e2922002-0218-4513-a62d-1da2ba64ee4c", + | "user_name": "user", + | "email": "$mail" + |} """.stripMargin HttpResponse(entity = HttpEntity(ContentType(`application/json`), response)) } @@ -121,7 +126,7 @@ class CloudFoundryUAAOAuth2AuthenticatorSpec extends FlatSpec with ScalatestRout } val userFuture = uaa.authenticate(code) - val user = Await.result(userFuture, 30 seconds) + val user = Await.result(userFuture, 30.seconds) assert(user.user == mail) assert(user.permissionLevel == Authenticator.User.permissionLevel) } diff --git a/services/jvm/src/test/scala/io/gearpump/services/security/oauth2/GoogleOAuth2AuthenticatorSpec.scala b/services/jvm/src/test/scala/io/gearpump/services/security/oauth2/GoogleOAuth2AuthenticatorSpec.scala index 58b4a34ca..70d8bb03c 100644 --- a/services/jvm/src/test/scala/io/gearpump/services/security/oauth2/GoogleOAuth2AuthenticatorSpec.scala +++ b/services/jvm/src/test/scala/io/gearpump/services/security/oauth2/GoogleOAuth2AuthenticatorSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,21 +18,24 @@ package io.gearpump.services.security.oauth2 +import scala.collection.JavaConverters._ +import scala.concurrent.Await +import scala.concurrent.duration._ + import akka.actor.ActorSystem import akka.http.scaladsl.model.HttpEntity.Strict import akka.http.scaladsl.model.MediaTypes._ -import akka.http.scaladsl.model.Uri.{Query, Path} +import akka.http.scaladsl.model.Uri.{Path, Query} import akka.http.scaladsl.model._ import akka.http.scaladsl.testkit.ScalatestRouteTest import com.typesafe.config.ConfigFactory -import io.gearpump.security.Authenticator -import io.gearpump.services.security.oauth2.GoogleOAuth2AuthenticatorSpec.MockGoogleAuthenticator -import io.gearpump.services.security.oauth2.impl.{GoogleOAuth2Authenticator, CloudFoundryUAAOAuth2Authenticator} import org.scalatest.FlatSpec -import scala.collection.JavaConverters._ -import scala.concurrent.Await -import scala.concurrent.duration._ +import io.gearpump.security.Authenticator +// NOTE: This cannot be removed!!! +import io.gearpump.services.util.UpickleUtil._ +import io.gearpump.services.security.oauth2.GoogleOAuth2AuthenticatorSpec.MockGoogleAuthenticator +import io.gearpump.services.security.oauth2.impl.GoogleOAuth2Authenticator class GoogleOAuth2AuthenticatorSpec extends FlatSpec with ScalatestRouteTest { @@ -71,7 +74,7 @@ class GoogleOAuth2AuthenticatorSpec extends FlatSpec with ScalatestRouteTest { val refreshToken = "eyJhbGciOiJSUzI1NiJ9.eyJqdGkiOiI2Nm" val mail = "test@gearpump.io" - def accessTokenEndpoint(request: HttpRequest) = { + def accessTokenEndpoint(request: HttpRequest): HttpResponse = { assert(request.entity.contentType.mediaType.value == "application/x-www-form-urlencoded") @@ -85,33 +88,37 @@ class GoogleOAuth2AuthenticatorSpec extends FlatSpec with ScalatestRouteTest { assert(form("redirect_uri") == configMap("callback")) assert(form("scope") == GoogleOAuth2Authenticator.Scope) - val response = s""" - |{ - | "access_token": "$accessToken", - | "token_type": "Bearer", - | "expires_in": 3591, - | "id_token": "eyJhbGciOiJSUzI1NiIsImtpZCI6ImY1NjQyYzY2MzdhYWQyOTJiOThlOGIwN2MwMzIxN2QwMzBmOTdkODkifQ.eyJpc3" - |} + // scalastyle:off line.size.limit + val response = + s""" + |{ + | "access_token": "$accessToken", + | "token_type": "Bearer", + | "expires_in": 3591, + | "id_token": "eyJhbGciOiJSUzI1NiIsImtpZCI6ImY1NjQyYzY2MzdhYWQyOTJiOThlOGIwN2MwMzIxN2QwMzBmOTdkODkifQ.eyJpc3" + |} """.stripMargin + // scalastyle:on line.size.limit HttpResponse(entity = HttpEntity(ContentType(`application/json`), response)) } - def protectedResourceEndpoint(request: HttpRequest) = { + def protectedResourceEndpoint(request: HttpRequest): HttpResponse = { assert(request.getUri().query().get("access_token").get == accessToken) - val response =s""" - |{ - | "kind": "plus#person", - | "etag": "4OZ_Kt6ujOh1jaML_U6RM6APqoE/mZ57HcMOYXaNXYXS5XEGJ9yVsI8", - | "nickname": "gearpump", - | "gender": "female", - | "emails": [ - | { - | "value": "$mail", - | "type": "account" - | } - | ] - | } + val response = + s""" + |{ + | "kind": "plus#person", + | "etag": "4OZ_Kt6ujOh1jaML_U6RM6APqoE/mZ57HcMOYXaNXYXS5XEGJ9yVsI8", + | "nickname": "gearpump", + | "gender": "female", + | "emails": [ + | { + | "value": "$mail", + | "type": "account" + | } + | ] + | } """.stripMargin HttpResponse(entity = HttpEntity(ContentType(`application/json`), response)) } @@ -127,7 +134,7 @@ class GoogleOAuth2AuthenticatorSpec extends FlatSpec with ScalatestRouteTest { } val userFuture = google.authenticate(code) - val user = Await.result(userFuture, 30 seconds) + val user = Await.result(userFuture, 30.seconds) assert(user.user == mail) assert(user.permissionLevel == Authenticator.Guest.permissionLevel) } diff --git a/services/jvm/src/test/scala/io/gearpump/services/security/oauth2/MockOAuth2Server.scala b/services/jvm/src/test/scala/io/gearpump/services/security/oauth2/MockOAuth2Server.scala index 9b6b5e948..c03532d30 100644 --- a/services/jvm/src/test/scala/io/gearpump/services/security/oauth2/MockOAuth2Server.scala +++ b/services/jvm/src/test/scala/io/gearpump/services/security/oauth2/MockOAuth2Server.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,18 +18,18 @@ package io.gearpump.services.security.oauth2 -import akka.actor.{ActorRef, ActorSystem} +import scala.concurrent.{Await, Future} + +import akka.actor.ActorSystem +import akka.http.scaladsl.Http import akka.http.scaladsl.Http.ServerBinding -import akka.http.scaladsl.model.{HttpResponse, HttpRequest} -import akka.http.scaladsl.server.Route +import akka.http.scaladsl.model.{HttpRequest, HttpResponse} import akka.stream.ActorMaterializer -import akka.http.scaladsl.Http -import akka.http.scaladsl.server.Directives._ -import akka.stream.scaladsl.{Sink, Source} -import io.gearpump.util.Util -import akka.pattern.ask +import akka.stream.scaladsl.Sink -import scala.concurrent.{Await, Future} +import io.gearpump.util.Util +// NOTE: This cannot be removed!! +import io.gearpump.services.util.UpickleUtil._ /** * Serves as a fake OAuth2 server. @@ -48,12 +48,11 @@ class MockOAuth2Server( def port: Int = _port def start(): Unit = { - _port = Util.findFreePort.get + _port = Util.findFreePort().get val serverSource = Http().bind(interface = "127.0.0.1", port = _port) bindingFuture = { serverSource.to(Sink.foreach { connection => - println("Accepted new connection from " + connection.remoteAddress) connection handleWithSyncHandler requestHandler }).run() } @@ -61,6 +60,6 @@ class MockOAuth2Server( def stop(): Unit = { import scala.concurrent.duration._ - Await.result(bindingFuture.map(_.unbind()), 120 seconds) + Await.result(bindingFuture.map(_.unbind()), 120.seconds) } } \ No newline at end of file diff --git a/services/jvm/src/test/scala/io/gearpump/services/util/UpickleSpec.scala b/services/jvm/src/test/scala/io/gearpump/services/util/UpickleSpec.scala index 9b3069ea6..b07481368 100644 --- a/services/jvm/src/test/scala/io/gearpump/services/util/UpickleSpec.scala +++ b/services/jvm/src/test/scala/io/gearpump/services/util/UpickleSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,14 +18,15 @@ package io.gearpump.services.util +import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} +import upickle.default.{read, write} + import io.gearpump.cluster.UserConfig import io.gearpump.metrics.Metrics.{Counter, MetricType} import io.gearpump.services.util.UpickleUtil._ import io.gearpump.streaming.ProcessorId import io.gearpump.streaming.appmaster.{ProcessorSummary, StreamAppMasterSummary} import io.gearpump.util.Graph -import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} -import upickle.default.{read, write} class UpickleSpec extends FlatSpec with Matchers with BeforeAndAfterEach { diff --git a/streaming/README.md b/streaming/README.md deleted file mode 100644 index 86d4ec2e0..000000000 --- a/streaming/README.md +++ /dev/null @@ -1,116 +0,0 @@ -A very basic DSL which support flatmap, reduce, and etc.. -====================== - - -Supported operators: ------------------- -```scala -class Stream[T](dag: Graph[Op, OpEdge], private val thisNode: Op, private val edge: Option[OpEdge] = None) { - - /** - * convert a value[T] to a list of value[R] - * @param fun - * @tparam R - * @return - */ - def flatMap[R](fun: T => TraversableOnce[R]): Stream[R] - - /** - * convert value[T] to value[R] - * @param fun - * @tparam R - * @return - */ - def map[R](fun: T => R): Stream[R] - - /** - * reserve records when fun(T) == true - * @param fun - * @return - */ - def filter(fun: T => Boolean): Stream[T] - - /** - * Reduce opeartion - * @param fun - * @return - */ - def reduce(fun: (T, T) => T): Stream[T] - - /** - * Log to task log file - */ - def log(): Unit - - /** - * Merge data from two stream into one - * @param other - * @return - */ - def merge(other: Stream[T]): Stream[T] - - /** - * Group by fun(T) - * - * For example, we have T type, People(name: String, gender: String, age: Int) - * groupBy[People](_.gender) will group the people by gender. - * - * You can append other combinators after groupBy - * - * For example, - * - * Stream[People].groupBy(_.gender).flatmap(..).filter.(..).reduce(..) - * - * @param fun - * @param parallism - * @tparam Group - * @return - */ - def groupBy[Group](fun: T => Group, parallism: Int = 1): Stream[T] - - /** - * connect with a low level Processor(TaskDescription) - * @param processor - * @param parallism - * @tparam R - * @return - */ - def process[R](processor: Class[_ <: Task], parallism: Int): Stream[R] -} - -``` - -How to define the DSL ---------------- -WordCount: - -```scala -val context = ClientContext(master) - val app = StreamApp("dsl", context) - - val data = "This is a good start, bingo!! bingo!!" - app.fromCollection(data.lines.toList). - // word => (word, count) - flatMap(line => line.split("[\\s]+")).map((_, 1)). - // (word, count1), (word, count2) => (word, count1 + count2) - groupBy(kv => kv._1).reduce((left, right) => (left._1, left._2 + right._2)) - - val appId = context.submit(app) - context.close() -``` - -For the full example, please check https://github.com/intel-hadoop/gearpump/tree/master/experiments/dsl/src/main/scala/io.gearpump/streaming/dsl/example - - -Run an example ---------------------- -```bash -# start master -bin\local - -# start UI -bin\services - -# start example topology -bin\gear io.gearpump.streaming.dsl.example.WordCount -``` \ No newline at end of file diff --git a/streaming/src/main/java/io/gearpump/streaming/javaapi/Graph.java b/streaming/src/main/java/io/gearpump/streaming/javaapi/Graph.java index 8b81253d6..dbae1c464 100644 --- a/streaming/src/main/java/io/gearpump/streaming/javaapi/Graph.java +++ b/streaming/src/main/java/io/gearpump/streaming/javaapi/Graph.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,8 +19,15 @@ package io.gearpump.streaming.javaapi; import io.gearpump.partitioner.Partitioner; +import io.gearpump.streaming.Processor; +import io.gearpump.streaming.task.Task; -public class Graph extends io.gearpump.util.Graph, Partitioner> { +/** + * Java version of Graph + * + * See {@link io.gearpump.util.Graph} + */ +public class Graph extends io.gearpump.util.Graph, Partitioner> { public Graph() { super(null, null); diff --git a/streaming/src/main/java/io/gearpump/streaming/javaapi/Processor.java b/streaming/src/main/java/io/gearpump/streaming/javaapi/Processor.java index 52b5d61de..974183eb9 100644 --- a/streaming/src/main/java/io/gearpump/streaming/javaapi/Processor.java +++ b/streaming/src/main/java/io/gearpump/streaming/javaapi/Processor.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -27,6 +27,11 @@ import io.gearpump.streaming.source.DataSourceProcessor; import io.gearpump.streaming.source.DataSourceTask; +/** + * Java version of Processor + * + * See {@link io.gearpump.streaming.Processor} + */ public class Processor implements io.gearpump.streaming.Processor { private Class _taskClass; private int _parallelism = 1; @@ -43,46 +48,49 @@ public Processor(Class taskClass, int parallelism) { } /** - * Create a Sink Processor - * @param dataSink the data sink itself - * @param parallelism the parallelism of this processor - * @param description the description for this processor - * @param taskConf the configuration for this processor - * @param system actor system - * @return the new created sink processor + * Creates a Sink Processor + * + * @param dataSink the data sink itself + * @param parallelism the parallelism of this processor + * @param description the description for this processor + * @param taskConf the configuration for this processor + * @param system actor system + * @return the new created sink processor */ - public static Processor sink(DataSink dataSink, int parallelism, String description, UserConfig taskConf, ActorSystem system) { + public static Processor sink(DataSink dataSink, int parallelism, String description, UserConfig taskConf, ActorSystem system) { io.gearpump.streaming.Processor p = DataSinkProcessor.apply(dataSink, parallelism, description, taskConf, system); return new Processor(p); } /** - * Create a Source Processor - * @param source the data source itself - * @param parallelism the parallelism of this processor - * @param description the description of this processor - * @param taskConf the configuration of this processor - * @param system actor system - * @return the new created source processor + * Creates a Source Processor + * + * @param source the data source itself + * @param parallelism the parallelism of this processor + * @param description the description of this processor + * @param taskConf the configuration of this processor + * @param system actor system + * @return the new created source processor */ - public static Processor source(DataSource source, int parallelism, String description, UserConfig taskConf, ActorSystem system) { + public static Processor source(DataSource source, int parallelism, String description, UserConfig taskConf, ActorSystem system) { io.gearpump.streaming.Processor p = DataSourceProcessor.apply(source, parallelism, description, taskConf, system); return new Processor(p); } public Processor(io.gearpump.streaming.Processor processor) { - this._taskClass = (Class)(processor.taskClass()); + this._taskClass = (Class) (processor.taskClass()); this._parallelism = processor.parallelism(); this._description = processor.description(); this._userConf = processor.taskConf(); } /** - * Create a general processor with user specified task logic. - * @param taskClass task implementation class of this processor (shall be a derived class from {@link Task} + * Creates a general processor with user specified task logic. + * + * @param taskClass task implementation class of this processor (shall be a derived class from {@link Task} * @param parallelism, how many initial tasks you want to use * @param description, some text to describe this processor - * @param taskConf, Processor specific configuration + * @param taskConf, Processor specific configuration */ public Processor(Class taskClass, int parallelism, String description, UserConfig taskConf) { this._taskClass = taskClass; diff --git a/streaming/src/main/java/io/gearpump/streaming/javaapi/StreamApplication.java b/streaming/src/main/java/io/gearpump/streaming/javaapi/StreamApplication.java index d01779309..150a26fd4 100644 --- a/streaming/src/main/java/io/gearpump/streaming/javaapi/StreamApplication.java +++ b/streaming/src/main/java/io/gearpump/streaming/javaapi/StreamApplication.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,7 +16,6 @@ * limitations under the License. */ - package io.gearpump.streaming.javaapi; import akka.actor.ActorSystem; @@ -24,21 +23,25 @@ import io.gearpump.cluster.ApplicationMaster; import io.gearpump.cluster.UserConfig; - +/** + * Java version of StreamApplication. + * + * Also see {@link io.gearpump.streaming.StreamApplication} + */ public class StreamApplication implements Application { private io.gearpump.streaming.StreamApplication app; /** - * Create a streaming application - * @param name name of the application - * @param conf user configuration - * @param graph the DAG + * Creates a streaming application * + * @param name Name of the application + * @param conf User configuration + * @param graph The DAG */ public StreamApplication(String name, UserConfig conf, Graph graph) { //by pass the tricky type check in scala 2.10 io.gearpump.util.Graph untypedGraph = graph; this.app = io.gearpump.streaming.StreamApplication.apply( - name, untypedGraph, conf); + name, untypedGraph, conf); } @Override diff --git a/streaming/src/main/java/io/gearpump/streaming/javaapi/Task.java b/streaming/src/main/java/io/gearpump/streaming/javaapi/Task.java index 836287d11..45fae19f4 100644 --- a/streaming/src/main/java/io/gearpump/streaming/javaapi/Task.java +++ b/streaming/src/main/java/io/gearpump/streaming/javaapi/Task.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -24,6 +24,11 @@ import io.gearpump.streaming.task.StartTime; import io.gearpump.streaming.task.TaskContext; +/** + * Java version of Task. + * + * See {@link io.gearpump.streaming.task.Task} + */ public class Task extends io.gearpump.streaming.task.Task { protected TaskContext context; protected UserConfig userConf; @@ -40,11 +45,14 @@ final public ActorRef self() { } @Override - public void onStart(StartTime startTime){} + public void onStart(StartTime startTime) { + } @Override - public void onNext(Message msg){} + public void onNext(Message msg) { + } @Override - public void onStop(){} + public void onStop() { + } } \ No newline at end of file diff --git a/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/FilterFunction.java b/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/FilterFunction.java index e4e137ff3..bb9744257 100644 --- a/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/FilterFunction.java +++ b/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/FilterFunction.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,7 +21,9 @@ import java.io.Serializable; /** - * a function that decides whether to reserve a value + * Filter function + * + * @param Message of type T */ public interface FilterFunction extends Serializable { boolean apply(T t); diff --git a/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/FlatMapFunction.java b/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/FlatMapFunction.java index b65a33823..3e18cf1b9 100644 --- a/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/FlatMapFunction.java +++ b/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/FlatMapFunction.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -22,7 +22,10 @@ import java.util.Iterator; /** - * a function that converts a value to a iterator of value + * Function that converts a value of type T to a iterator of values of type R. + * + * @param Input value type + * @param Return value type */ public interface FlatMapFunction extends Serializable { Iterator apply(T t); diff --git a/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/GroupByFunction.java b/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/GroupByFunction.java index 651c477e0..2ba524eab 100644 --- a/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/GroupByFunction.java +++ b/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/GroupByFunction.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,7 +21,10 @@ import java.io.Serializable; /** - * a function that puts a value into a Group + * GroupBy function which assign value of type T to groups + * + * @param Input value type + * @param Group Type */ public interface GroupByFunction extends Serializable { Group apply(T t); diff --git a/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/MapFunction.java b/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/MapFunction.java index a30a671a2..b4bd6ac8d 100644 --- a/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/MapFunction.java +++ b/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/MapFunction.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,7 +21,10 @@ import java.io.Serializable; /** - * a function that converts a value to value + * Function that map a value of type T to value of type R + * + * @param Input value type + * @param Output value type */ public interface MapFunction extends Serializable { R apply(T t); diff --git a/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/ReduceFunction.java b/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/ReduceFunction.java index 0f4bb1876..f439c0aa4 100644 --- a/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/ReduceFunction.java +++ b/streaming/src/main/java/io/gearpump/streaming/javaapi/dsl/functions/ReduceFunction.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,7 +21,9 @@ import java.io.Serializable; /** - * a function that applies reduce operation + * Function that applies reduce operation + * + * @param Input value type */ public interface ReduceFunction extends Serializable { T apply(T t1, T t2); diff --git a/streaming/src/main/scala/io/gearpump/streaming/ClusterMessage.scala b/streaming/src/main/scala/io/gearpump/streaming/ClusterMessage.scala index 370b36cb3..06cf022be 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/ClusterMessage.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/ClusterMessage.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,29 +18,34 @@ package io.gearpump.streaming +import scala.language.existentials + import akka.actor.ActorRef -import io.gearpump.streaming.appmaster.TaskRegistry.TaskLocations -import io.gearpump.streaming.task.{TaskId, Subscriber} + import io.gearpump.TimeStamp import io.gearpump.cluster.appmaster.WorkerInfo import io.gearpump.cluster.scheduler.Resource +import io.gearpump.streaming.appmaster.TaskRegistry.TaskLocations +import io.gearpump.streaming.task.{Subscriber, TaskId} import io.gearpump.transport.HostPort -import scala.language.existentials - object AppMasterToExecutor { - case class LaunchTasks(taskId: List[TaskId], dagVersion: Int, processorDescription: ProcessorDescription, subscribers: List[Subscriber]) + case class LaunchTasks( + taskId: List[TaskId], dagVersion: Int, processorDescription: ProcessorDescription, + subscribers: List[Subscriber]) case object TasksLaunched /** * dagVersion, life, and subscribers will be changed on target task list. */ - case class ChangeTasks(taskId: List[TaskId], dagVersion: Int, life: LifeTime, subscribers: List[Subscriber]) + case class ChangeTasks( + taskId: List[TaskId], dagVersion: Int, life: LifeTime, subscribers: List[Subscriber]) case class TasksChanged(taskIds: List[TaskId]) - case class ChangeTask(taskId: TaskId, dagVersion: Int, life: LifeTime, subscribers: List[Subscriber]) + case class ChangeTask( + taskId: TaskId, dagVersion: Int, life: LifeTime, subscribers: List[Subscriber]) case class TaskChanged(taskId: TaskId, dagVersion: Int) @@ -52,7 +57,8 @@ object AppMasterToExecutor { case class TaskLocationsReceived(dagVersion: Int, executorId: ExecutorId) - case class TaskLocationsRejected(dagVersion: Int, executorId: ExecutorId, reason: String, ex: Throwable) + case class TaskLocationsRejected( + dagVersion: Int, executorId: ExecutorId, reason: String, ex: Throwable) case class StartAllTasks(dagVersion: Int) @@ -65,10 +71,11 @@ object AppMasterToExecutor { } object ExecutorToAppMaster { - case class RegisterExecutor(executor: ActorRef, executorId: Int, resource: Resource, worker : WorkerInfo) + case class RegisterExecutor( + executor: ActorRef, executorId: Int, resource: Resource, worker : WorkerInfo) - case class RegisterTask(taskId: TaskId, executorId : Int, task: HostPort) - case class UnRegisterTask(taskId: TaskId, executorId : Int) + case class RegisterTask(taskId: TaskId, executorId: Int, task: HostPort) + case class UnRegisterTask(taskId: TaskId, executorId: Int) case class MessageLoss(executorId: Int, taskId: TaskId, cause: String) } diff --git a/streaming/src/main/scala/io/gearpump/streaming/Constants.scala b/streaming/src/main/scala/io/gearpump/streaming/Constants.scala index fa23807ab..becf31ac5 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/Constants.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/Constants.scala @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.gearpump.streaming object Constants { @@ -11,6 +29,9 @@ object Constants { val GEARPUMP_STREAMING_REGISTER_TASK_TIMEOUT_MS = "gearpump.streaming.register-task-timeout-ms" - val GEARPUMP_STREAMING_MAX_PENDING_MESSAGE_COUNT = "gearpump.streaming.max-pending-message-count-per-connection" - val GEARPUMP_STREAMING_ACK_ONCE_EVERY_MESSAGE_COUNT = "gearpump.streaming.ack-once-every-message-count" + val GEARPUMP_STREAMING_MAX_PENDING_MESSAGE_COUNT = + "gearpump.streaming.max-pending-message-count-per-connection" + + val GEARPUMP_STREAMING_ACK_ONCE_EVERY_MESSAGE_COUNT = + "gearpump.streaming.ack-once-every-message-count" } diff --git a/streaming/src/main/scala/io/gearpump/streaming/DAG.scala b/streaming/src/main/scala/io/gearpump/streaming/DAG.scala index a6b8cbaaa..a73fd4855 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/DAG.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/DAG.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,14 +18,16 @@ package io.gearpump.streaming -import io.gearpump.streaming.task.TaskId import io.gearpump.partitioner.PartitionerDescription +import io.gearpump.streaming.task.TaskId import io.gearpump.util.Graph /** - * DAG is wrapper for [[Graph]] for streaming applications. + * DAG is wrapper for [[io.gearpump.util.Graph]] for streaming applications. */ -case class DAG(version: Int, processors : Map[ProcessorId, ProcessorDescription], graph : Graph[ProcessorId, PartitionerDescription]) extends Serializable { +case class DAG(version: Int, processors : Map[ProcessorId, ProcessorDescription], + graph : Graph[ProcessorId, PartitionerDescription]) + extends Serializable { def isEmpty: Boolean = { processors.isEmpty @@ -46,15 +48,15 @@ case class DAG(version: Int, processors : Map[ProcessorId, ProcessorDescription] } object DAG { - def apply (graph : Graph[ProcessorDescription, PartitionerDescription], version: Int = 0) : DAG = { - val processors = graph.vertices.map{processorDescription => + def apply(graph: Graph[ProcessorDescription, PartitionerDescription], version: Int = 0): DAG = { + val processors = graph.vertices.map { processorDescription => (processorDescription.id, processorDescription) }.toMap - val dag = graph.mapVertex{ processor => + val dag = graph.mapVertex { processor => processor.id } new DAG(version, processors, dag) } - def empty() = apply(Graph.empty[ProcessorDescription, PartitionerDescription]) + def empty: DAG = apply(Graph.empty[ProcessorDescription, PartitionerDescription]) } \ No newline at end of file diff --git a/streaming/src/main/scala/io/gearpump/streaming/MessageSerializer.scala b/streaming/src/main/scala/io/gearpump/streaming/MessageSerializer.scala index 4eabcd2f6..8eb866d9b 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/MessageSerializer.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/MessageSerializer.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -75,7 +75,7 @@ class InitialAckRequestSerializer extends TaskMessageSerializer[InitialAckReques } } -class AckRequestSerializer extends TaskMessageSerializer[AckRequest]{ +class AckRequestSerializer extends TaskMessageSerializer[AckRequest] { val taskIdSerializer = new TaskIdSerializer override def getLength(obj: AckRequest): Int = taskIdSerializer.getLength(obj.taskId) + 6 diff --git a/streaming/src/main/scala/io/gearpump/streaming/StreamApplication.scala b/streaming/src/main/scala/io/gearpump/streaming/StreamApplication.scala index b68054a70..0ca4d9242 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/StreamApplication.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/StreamApplication.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,20 +18,20 @@ package io.gearpump.streaming +import scala.language.implicitConversions +import scala.reflect.ClassTag + import akka.actor.ActorSystem -import io.gearpump.streaming.appmaster.AppMaster -import io.gearpump.streaming.task.Task + import io.gearpump.TimeStamp import io.gearpump.cluster._ import io.gearpump.partitioner.{HashPartitioner, Partitioner, PartitionerDescription, PartitionerObject} -import io.gearpump.util.{LogUtil, Graph, ReferenceEqual} - -import scala.language.implicitConversions -import scala.reflect.ClassTag +import io.gearpump.streaming.appmaster.AppMaster +import io.gearpump.streaming.task.Task +import io.gearpump.util.{Graph, LogUtil, ReferenceEqual} /** * Processor is the blueprint for tasks. - * */ trait Processor[+T <: Task] extends ReferenceEqual { @@ -41,7 +41,7 @@ trait Processor[+T <: Task] extends ReferenceEqual { def parallelism: Int /** - * The custom [[UserConfig]], it is used to initialize a task in runtime. + * The custom [[io.gearpump.cluster.UserConfig]], it is used to initialize a task in runtime. */ def taskConf: UserConfig @@ -59,20 +59,29 @@ trait Processor[+T <: Task] extends ReferenceEqual { } object Processor { - def ProcessorToProcessorDescription(id: ProcessorId, processor: Processor[_ <: Task]): ProcessorDescription = { + def ProcessorToProcessorDescription(id: ProcessorId, processor: Processor[_ <: Task]) + : ProcessorDescription = { import processor._ ProcessorDescription(id, taskClass.getName, parallelism, description, taskConf) } - def apply[T<: Task](parallelism : Int, description: String = "", taskConf: UserConfig = UserConfig.empty)(implicit classtag: ClassTag[T]): DefaultProcessor[T] = { - new DefaultProcessor[T](parallelism, description, taskConf, classtag.runtimeClass.asInstanceOf[Class[T]]) + def apply[T<: Task]( + parallelism : Int, description: String = "", + taskConf: UserConfig = UserConfig.empty)(implicit classtag: ClassTag[T]) + : DefaultProcessor[T] = { + new DefaultProcessor[T](parallelism, description, taskConf, + classtag.runtimeClass.asInstanceOf[Class[T]]) } - def apply[T<: Task](taskClazz: Class[T], parallelism : Int, description: String, taskConf: UserConfig): DefaultProcessor[T] = { + def apply[T<: Task]( + taskClazz: Class[T], parallelism : Int, description: String, taskConf: UserConfig) + : DefaultProcessor[T] = { new DefaultProcessor[T](parallelism, description, taskConf, taskClazz) } - case class DefaultProcessor[T<: Task](parallelism : Int, description: String, taskConf: UserConfig, taskClass: Class[T]) extends Processor[T] { + case class DefaultProcessor[T<: Task]( + parallelism : Int, description: String, taskConf: UserConfig, taskClass: Class[T]) + extends Processor[T] { def withParallelism(parallel: Int): DefaultProcessor[T] = { new DefaultProcessor[T](parallel, description, taskConf, taskClass) @@ -93,7 +102,6 @@ object Processor { * * When input message's timestamp is beyond current processor's lifetime, * then it will not be processed by this processor. - * */ case class LifeTime(birth: TimeStamp, death: TimeStamp) { def contains(timestamp: TimeStamp): Boolean = { @@ -112,7 +120,9 @@ object LifeTime { /** * Represent a streaming application */ -class StreamApplication(override val name : String, val inputUserConfig: UserConfig, val dag: Graph[ProcessorDescription, PartitionerDescription]) +class StreamApplication( + override val name: String, val inputUserConfig: UserConfig, + val dag: Graph[ProcessorDescription, PartitionerDescription]) extends Application { require(!dag.hasDuplicatedEdge(), "Graph should not have duplicated edges") @@ -137,19 +147,21 @@ object StreamApplication { private val hashPartitioner = new HashPartitioner() private val LOG = LogUtil.getLogger(getClass) - def apply[T <: Processor[Task], P <: Partitioner] (name : String, dag: Graph[T, P], userConfig: UserConfig): StreamApplication = { - import Processor._ + def apply[T <: Processor[Task], P <: Partitioner]( + name: String, dag: Graph[T, P], userConfig: UserConfig): StreamApplication = { + import io.gearpump.streaming.Processor._ if (dag.hasCycle()) { LOG.warn(s"Detected cycles in DAG of application $name!") } val indices = dag.topologicalOrderWithCirclesIterator.toList.zipWithIndex.toMap - val graph = dag.mapVertex {processor => + val graph = dag.mapVertex { processor => val updatedProcessor = ProcessorToProcessorDescription(indices(processor), processor) updatedProcessor }.mapEdge { (node1, edge, node2) => - PartitionerDescription(new PartitionerObject(Option(edge).getOrElse(StreamApplication.hashPartitioner))) + PartitionerDescription(new PartitionerObject( + Option(edge).getOrElse(StreamApplication.hashPartitioner))) } new StreamApplication(name, userConfig, graph) } diff --git a/streaming/src/main/scala/io/gearpump/streaming/appmaster/AppMaster.scala b/streaming/src/main/scala/io/gearpump/streaming/appmaster/AppMaster.scala index a3f2fc57f..df8501741 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/appmaster/AppMaster.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/appmaster/AppMaster.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,17 +19,21 @@ package io.gearpump.streaming.appmaster import java.lang.management.ManagementFactory +import scala.concurrent.Future import akka.actor._ +import org.slf4j.Logger + import io.gearpump._ import io.gearpump.cluster.ClientToMaster.{GetLastFailure, GetStallingTasks, QueryHistoryMetrics, ShutdownApplication} import io.gearpump.cluster.MasterToAppMaster.{AppMasterDataDetailRequest, ReplayFromTimestampWindowTrailingEdge} -import io.gearpump.cluster.MasterToClient.{HistoryMetricsItem, HistoryMetrics, LastFailure} +import io.gearpump.cluster.MasterToClient.{HistoryMetrics, HistoryMetricsItem, LastFailure} import io.gearpump.cluster._ +import io.gearpump.cluster.worker.WorkerId import io.gearpump.metrics.Metrics.ReportMetrics import io.gearpump.metrics.{JvmMetricsSet, Metrics, MetricsReporterService} import io.gearpump.partitioner.PartitionerDescription -import io.gearpump.streaming.ExecutorToAppMaster.{UnRegisterTask, MessageLoss, RegisterExecutor, RegisterTask} +import io.gearpump.streaming.ExecutorToAppMaster.{MessageLoss, RegisterExecutor, RegisterTask, UnRegisterTask} import io.gearpump.streaming._ import io.gearpump.streaming.appmaster.AppMaster._ import io.gearpump.streaming.appmaster.DagManager.{GetLatestDAG, LatestDAG, ReplaceProcessor} @@ -42,30 +46,27 @@ import io.gearpump.streaming.util.ActorPathUtil import io.gearpump.util.Constants.{APPMASTER_DEFAULT_EXECUTOR_ID, _} import io.gearpump.util.HistoryMetricsService.HistoryMetricsConfig import io.gearpump.util._ -import org.slf4j.Logger - -import scala.concurrent.Future /** * AppMaster is the head of a streaming application. * * It contains: - * 1. ExecutorManager to manage all executors. - * 2. TaskManager to manage all tasks, - * 3. ClockService to track the global clock for this streaming application. - * 4. Scheduler to decide which a task should be scheduled to. - * + * 1. ExecutorManager to manage all executors. + * 2. TaskManager to manage all tasks, + * 3. ClockService to track the global clock for this streaming application. + * 4. Scheduler to decide which a task should be scheduled to. */ -class AppMaster(appContext : AppMasterContext, app : AppDescription) extends ApplicationMaster { +class AppMaster(appContext: AppMasterContext, app: AppDescription) extends ApplicationMaster { import app.userConfig import appContext.{appId, masterProxy, username} - implicit val actorSystem = context.system - implicit val timeOut = FUTURE_TIMEOUT + private implicit val actorSystem = context.system + private implicit val timeOut = FUTURE_TIMEOUT + import akka.pattern.ask - implicit val dispatcher = context.dispatcher + private implicit val dispatcher = context.dispatcher - val startTime: TimeStamp = System.currentTimeMillis() + private val startTime: TimeStamp = System.currentTimeMillis() private val LOG: Logger = LogUtil.getLogger(getClass, app = appId) LOG.info(s"AppMaster[$appId] is launched by $username, app: $app xxxxxxxxxxxxxxxxx") @@ -83,11 +84,12 @@ class AppMaster(appContext : AppMasterContext, app : AppDescription) extends Ap private var lastFailure = LastFailure(0L, null) private val appMasterBrief = ExecutorBrief(APPMASTER_DEFAULT_EXECUTOR_ID, - self.path.toString, Option(appContext.workerInfo).map(_.workerId).getOrElse(WorkerId.unspecified), "active") + self.path.toString, + Option(appContext.workerInfo).map(_.workerId).getOrElse(WorkerId.unspecified), "active") private val getHistoryMetricsConfig = HistoryMetricsConfig(systemConfig) - val metricsEnabled = systemConfig.getBoolean(GEARPUMP_METRIC_ENABLED) + private val metricsEnabled = systemConfig.getBoolean(GEARPUMP_METRIC_ENABLED) private val userDir = System.getProperty("user.dir") private val logFile = LogUtil.applicationLogDir(actorSystem.settings.config) @@ -104,12 +106,15 @@ class AppMaster(appContext : AppMasterContext, app : AppDescription) extends Ap ) private val historyMetricsService = if (metricsEnabled) { - // register jvm metrics - Metrics(context.system).register(new JvmMetricsSet(s"app${appId}.executor${APPMASTER_DEFAULT_EXECUTOR_ID}")) + // Registers jvm metrics + Metrics(context.system).register(new JvmMetricsSet( + s"app${appId}.executor${APPMASTER_DEFAULT_EXECUTOR_ID}")) - val historyMetricsService = context.actorOf(Props(new HistoryMetricsService(s"app$appId", getHistoryMetricsConfig))) + val historyMetricsService = context.actorOf(Props(new HistoryMetricsService( + s"app$appId", getHistoryMetricsConfig))) - val metricsReportService = context.actorOf(Props(new MetricsReporterService(Metrics(context.system)))) + val metricsReportService = context.actorOf(Props( + new MetricsReporterService(Metrics(context.system)))) historyMetricsService.tell(ReportMetrics, metricsReportService) Some(historyMetricsService) @@ -129,13 +134,15 @@ class AppMaster(appContext : AppMasterContext, app : AppDescription) extends Ap jarScheduler, executorManager, clockService.get, self, app.name)))) } - override def receive : Receive = - taskMessageHandler orElse + override def receive: Receive = { + taskMessageHandler orElse executorMessageHandler orElse recover orElse appMasterService orElse ActorUtil.defaultMsgHandler(self) + } + /** Handles messages from Tasks */ def taskMessageHandler: Receive = { case clock: ClockEvent => taskManager.foreach(_ forward clock) @@ -143,7 +150,7 @@ class AppMaster(appContext : AppMasterContext, app : AppDescription) extends Ap taskManager.foreach(_ forward register) case unRegister: UnRegisterTask => taskManager.foreach(_ forward unRegister) - // check whether this processor dead, if it is, then we should remove it from clockService. + // Checks whether this processor dead, if it is, then we should remove it from clockService. clockService.foreach(_ forward CheckProcessorDeath(unRegister.taskId.processorId)) case replay: ReplayFromTimestampWindowTrailingEdge => taskManager.foreach(_ forward replay) @@ -163,6 +170,7 @@ class AppMaster(appContext : AppMasterContext, app : AppDescription) extends Ap clockService.foreach(_ forward GetCheckpointClock) } + /** Handles messages from Executors */ def executorMessageHandler: Receive = { case register: RegisterExecutor => executorManager forward register @@ -170,6 +178,7 @@ class AppMaster(appContext : AppMasterContext, app : AppDescription) extends Ap historyMetricsService.foreach(_ forward ReportMetrics) } + /** Handles messages from AppMaster */ def appMasterService: Receive = { case appMasterDataDetailRequest: AppMasterDataDetailRequest => LOG.debug(s"AppMaster got AppMasterDataDetailRequest for $appId ") @@ -179,14 +188,17 @@ class AppMaster(appContext : AppMasterContext, app : AppDescription) extends Ap val taskFuture = getTaskList val dagFuture = getDAG - val appMasterDataDetail = for {executors <- executorsFuture + val appMasterDataDetail = for { + executors <- executorsFuture clock <- clockFuture tasks <- taskFuture dag <- dagFuture } yield { val graph = dag.graph - val executorToTasks = tasks.tasks.groupBy(_._2).mapValues {_.keys.toList} + val executorToTasks = tasks.tasks.groupBy(_._2).mapValues { + _.keys.toList + } val processors = dag.processors.map { kv => val processor = kv._2 @@ -195,7 +207,8 @@ class AppMaster(appContext : AppMasterContext, app : AppDescription) extends Ap (kv._1, TaskCount(kv._2.count(_.processorId == id))) }.filter(_._2.count != 0) (id, - ProcessorSummary(id, taskClass, parallelism, description, taskConf, life, tasks.keys.toList, tasks)) + ProcessorSummary(id, taskClass, parallelism, description, taskConf, life, + tasks.keys.toList, tasks)) } StreamAppMasterSummary( @@ -211,7 +224,7 @@ class AppMaster(appContext : AppMasterContext, app : AppDescription) extends Ap logFile = logFile.getAbsolutePath, processors = processors, processorLevels = graph.vertexHierarchyLevelMap(), - dag = graph.mapEdge {(node1, edge, node2) => + dag = graph.mapEdge { (node1, edge, node2) => edge.partitionerFactory.name }, executors = executors, @@ -221,16 +234,16 @@ class AppMaster(appContext : AppMasterContext, app : AppDescription) extends Ap val client = sender() - appMasterDataDetail.map{appData => + appMasterDataDetail.map { appData => client ! appData } -// TODO: WebSocket is buggy and disabled. -// case appMasterMetricsRequest: AppMasterMetricsRequest => -// val client = sender() -// actorSystem.eventStream.subscribe(client, classOf[MetricType]) + // TODO: WebSocket is buggy and disabled. + // case appMasterMetricsRequest: AppMasterMetricsRequest => + // val client = sender() + // actorSystem.eventStream.subscribe(client, classOf[MetricType]) case query: QueryHistoryMetrics => if (historyMetricsService.isEmpty) { - // return empty metrics so that we don't hang the UI + // Returns empty metrics so that we don't hang the UI sender ! HistoryMetrics(query.path, List.empty[HistoryMetricsItem]) } else { historyMetricsService.get forward query @@ -241,34 +254,37 @@ class AppMaster(appContext : AppMasterContext, app : AppDescription) extends Ap dagManager forward replaceDAG case GetLastFailure(_) => sender ! lastFailure - case get@ GetExecutorSummary(executorId) => + case get@GetExecutorSummary(executorId) => val client = sender if (executorId == APPMASTER_DEFAULT_EXECUTOR_ID) { client ! appMasterExecutorSummary } else { - ActorUtil.askActor[Map[ExecutorId, ExecutorInfo]](executorManager, GetExecutorInfo).map { map => - map.get(executorId).foreach { executor => - executor.executor.tell(get, client) + ActorUtil.askActor[Map[ExecutorId, ExecutorInfo]](executorManager, GetExecutorInfo) + .map { map => + map.get(executorId).foreach { executor => + executor.executor.tell(get, client) + } } - } } - case query@ QueryExecutorConfig(executorId) => + case query@QueryExecutorConfig(executorId) => val client = sender if (executorId == -1) { val systemConfig = context.system.settings.config sender ! ExecutorConfig(ClusterConfig.filterOutDefaultConfig(systemConfig)) } else { - ActorUtil.askActor[Map[ExecutorId, ExecutorInfo]](executorManager, GetExecutorInfo).map { map => - map.get(executorId).foreach { executor => - executor.executor.tell(query, client) + ActorUtil.askActor[Map[ExecutorId, ExecutorInfo]](executorManager, GetExecutorInfo) + .map { map => + map.get(executorId).foreach { executor => + executor.executor.tell(query, client) + } } - } } - } + } + /** Error handling */ def recover: Receive = { case FailedToRecover(errorMsg) => - if(context.children.toList.contains(sender())){ + if (context.children.toList.contains(sender())) { LOG.error(errorMsg) masterProxy ! ShutdownApplication(appId) } @@ -288,14 +304,15 @@ class AppMaster(appContext : AppMasterContext, app : AppDescription) extends Ap } private def executorBrief: Future[List[ExecutorBrief]] = { - ActorUtil.askActor[Map[ExecutorId, ExecutorInfo]](executorManager, GetExecutorInfo).map { infos => - infos.values.map { info => - ExecutorBrief(info.executorId, - info.executor.path.toSerializationFormat, - info.worker.workerId, - "active") - }.toList :+ appMasterBrief - } + ActorUtil.askActor[Map[ExecutorId, ExecutorInfo]](executorManager, GetExecutorInfo) + .map { infos => + infos.values.map { info => + ExecutorBrief(info.executorId, + info.executor.path.toSerializationFormat, + info.worker.workerId, + "active") + }.toList :+ appMasterBrief + } } private def getTaskList: Future[TaskList] = { @@ -312,10 +329,11 @@ class AppMaster(appContext : AppMasterContext, app : AppDescription) extends Ap } private def getUpdatedDAG(): DAG = { - val dag = DAG(userConfig.getValue[Graph[ProcessorDescription, PartitionerDescription]](StreamApplication.DAG).get) - val updated = dag.processors.map{ idAndProcessor => + val dag = DAG(userConfig.getValue[Graph[ProcessorDescription, + PartitionerDescription]](StreamApplication.DAG).get) + val updated = dag.processors.map { idAndProcessor => val (id, oldProcessor) = idAndProcessor - val newProcessor = if(oldProcessor.jar == null) { + val newProcessor = if (oldProcessor.jar == null) { oldProcessor.copy(jar = appContext.appJar.getOrElse(null)) } else { oldProcessor @@ -327,14 +345,18 @@ class AppMaster(appContext : AppMasterContext, app : AppDescription) extends Ap } object AppMaster { + + /** Master node doesn't return resource in time */ case object AllocateResourceTimeOut + /** Query task ActorRef by providing the taskId */ case class LookupTaskActorRef(taskId: TaskId) case class TaskActorRef(task: ActorRef) class ServiceNotAvailableException(reason: String) extends Exception(reason) - case class ExecutorBrief(executorId: ExecutorId, executor: String, workerId: WorkerId, status: String) + case class ExecutorBrief( + executorId: ExecutorId, executor: String, workerId: WorkerId, status: String) } \ No newline at end of file diff --git a/streaming/src/main/scala/io/gearpump/streaming/appmaster/ClockService.scala b/streaming/src/main/scala/io/gearpump/streaming/appmaster/ClockService.scala index 67fd592bf..bd18bdcaf 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/appmaster/ClockService.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/appmaster/ClockService.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,45 +21,47 @@ package io.gearpump.streaming.appmaster import java.util import java.util.Date import java.util.concurrent.TimeUnit +import scala.concurrent.Future +import scala.concurrent.duration.FiniteDuration +import scala.language.implicitConversions import akka.actor.{Actor, Cancellable, Stash} +import org.slf4j.Logger + +import io.gearpump.TimeStamp +import io.gearpump.cluster.ClientToMaster.GetStallingTasks import io.gearpump.google.common.primitives.Longs -import io.gearpump.streaming.task._ +import io.gearpump.streaming.AppMasterToMaster.StallingTasks import io.gearpump.streaming._ +import io.gearpump.streaming.appmaster.ClockService.HealthChecker.ClockValue +import io.gearpump.streaming.appmaster.ClockService._ import io.gearpump.streaming.storage.AppDataStore -import io.gearpump.TimeStamp -import io.gearpump.cluster.ClientToMaster.GetStallingTasks -import AppMasterToMaster.StallingTasks -import ClockService.HealthChecker.ClockValue -import ClockService._ +import io.gearpump.streaming.task._ import io.gearpump.util.LogUtil -import org.slf4j.Logger - -import scala.concurrent.Future -import scala.concurrent.duration.FiniteDuration -import scala.language.implicitConversions /** - * The clockService will maintain a global view of message timestamp in the application + * Maintains a global view of message timestamp in the application */ -class ClockService(private var dag : DAG, store: AppDataStore) extends Actor with Stash { +class ClockService(private var dag: DAG, store: AppDataStore) extends Actor with Stash { private val LOG: Logger = LogUtil.getLogger(getClass) import context.dispatcher private val healthChecker = new HealthChecker(stallingThresholdSeconds = 60) - private var healthCheckScheduler : Cancellable = null - private var snapshotScheduler : Cancellable = null + private var healthCheckScheduler: Cancellable = null + private var snapshotScheduler: Cancellable = null - override def receive = null + override def receive: Receive = null - override def preStart() : Unit = { + override def preStart(): Unit = { LOG.info("Initializing Clock service, get snapshotted StartClock ....") store.get(START_CLOCK).asInstanceOf[Future[TimeStamp]].map { clock => val startClock = Option(clock).getOrElse(0L) minCheckpointClock = Some(startClock) + // Recover the application by restarting from last persisted startClock. + // Only messge after startClock will be replayed. self ! StoredStartClock(startClock) LOG.info(s"Start Clock Retrieved, starting ClockService, startClock: $startClock") } @@ -67,12 +69,15 @@ class ClockService(private var dag : DAG, store: AppDataStore) extends Actor wit context.become(waitForStartClock) } - override def postStop() : Unit = { + override def postStop(): Unit = { Option(healthCheckScheduler).map(_.cancel) Option(snapshotScheduler).map(_.cancel) } - var clocks = Map.empty[ProcessorId, ProcessorClock] + // Keep track of clock value of all processors. + private var clocks = Map.empty[ProcessorId, ProcessorClock] + + // Each process can have multiple upstream processors. This keep track of the upstream clocks. private var upstreamClocks = Map.empty[ProcessorId, Array[ProcessorClock]] // We use Array instead of List for Performance consideration @@ -106,12 +111,12 @@ class ClockService(private var dag : DAG, store: AppDataStore) extends Actor wit private def recoverDag(dag: DAG, startClock: TimeStamp): Unit = { this.clocks = dag.processors.filter(startClock < _._2.life.death). map { pair => - val (processorId, processor) = pair - val parallelism = processor.parallelism - val clock = new ProcessorClock(processorId, processor.life, parallelism) - clock.init(startClock) - (processorId, clock) - } + val (processorId, processor) = pair + val parallelism = processor.parallelism + val clock = new ProcessorClock(processorId, processor.life, parallelism) + clock.init(startClock) + (processorId, clock) + } this.upstreamClocks = clocks.map { pair => val (processorId, processor) = pair @@ -150,7 +155,7 @@ class ClockService(private var dag : DAG, store: AppDataStore) extends Actor wit (processorId, upstreamClocks.toArray) } - // init the clock of all processors. + // Inits the clock of all processors. newClocks.map { pair => val (processorId, processorClock) = pair val upstreamClock = getUpStreamMinClock(processorId) @@ -174,11 +179,12 @@ class ClockService(private var dag : DAG, store: AppDataStore) extends Actor wit import context.dispatcher - //period report current clock - healthCheckScheduler = context.system.scheduler.schedule(new FiniteDuration(5, TimeUnit.SECONDS), + // Period report current clock + healthCheckScheduler = context.system.scheduler.schedule( + new FiniteDuration(5, TimeUnit.SECONDS), new FiniteDuration(60, TimeUnit.SECONDS), self, HealthCheck) - //period snpashot latest min startclock to external storage + // Period snpashot latest min startclock to external storage snapshotScheduler = context.system.scheduler.schedule(new FiniteDuration(5, TimeUnit.SECONDS), new FiniteDuration(5, TimeUnit.SECONDS), self, SnapshotStartClock) @@ -206,7 +212,7 @@ class ClockService(private var dag : DAG, store: AppDataStore) extends Actor wit case GetUpstreamMinClock(task) => sender ! UpstreamMinClock(getUpStreamMinClock(task.processorId)) - case update@ UpdateClock(task, clock) => + case update@UpdateClock(task, clock) => val upstreamMinClock = getUpStreamMinClock(task.processorId) val processorClock = clocks.get(task.processorId) @@ -233,7 +239,8 @@ class ClockService(private var dag : DAG, store: AppDataStore) extends Actor wit LOG.info(s"Removing $processorId from clock service...") removeProcessor(processorId) } else { - LOG.info(s"Unsuccessfully in removing $processorId from clock service..., min: ${processorClock.get.min}, life: $life") + LOG.info(s"Unsuccessfully in removing $processorId from clock service...," + + s" min: ${processorClock.get.min}, life: $life") } } case HealthCheck => @@ -253,15 +260,15 @@ class ClockService(private var dag : DAG, store: AppDataStore) extends Actor wit case ChangeToNewDAG(dag) => if (dag.version > this.dag.version) { - // transit to a new dag version + // Transits to a new dag version this.dag = dag dynamicDAG(dag, getStartClock) } else { - // restart current dag. + // Restarts current dag. recoverDag(dag, getStartClock) } - LOG.info(s"Change to new DAG(dag = ${dag.version}), send back ChangeToNewDAGSuccess") - sender ! ChangeToNewDAGSuccess(clocks.map{ pair => + LOG.info(s"Change to new DAG(dag = ${dag.version}), send back ChangeToNewDAGSuccess") + sender ! ChangeToNewDAGSuccess(clocks.map { pair => val (id, clock) = pair (id, clock.min) }) @@ -279,7 +286,7 @@ class ClockService(private var dag : DAG, store: AppDataStore) extends Actor wit upstreamClocks = upstreamClocks - processorId - // remove dead processor from checkpoints. + // Removes dead processor from checkpoints. checkpointClocks = checkpointClocks.filter { kv => val (taskId, processor) = kv taskId.processorId != processorId @@ -290,12 +297,13 @@ class ClockService(private var dag : DAG, store: AppDataStore) extends Actor wit ProcessorClocks.minClock(processorClocks) } - def selfCheck() : Unit = { + def selfCheck(): Unit = { val minTimestamp = minClock if (Long.MaxValue == minTimestamp) { processorClocks.foreach { clock => - LOG.info(s"Processor ${clock.processorId} Clock: min: ${clock.min}, taskClocks: "+ clock.taskClocks.mkString(",")) + LOG.info(s"Processor ${clock.processorId} Clock: min: ${clock.min}, " + + s"taskClocks: " + clock.taskClocks.mkString(",")) } } @@ -306,7 +314,7 @@ class ClockService(private var dag : DAG, store: AppDataStore) extends Actor wit minCheckpointClock.getOrElse(minClock) } - private def snapshotStartClock() : Unit = { + private def snapshotStartClock(): Unit = { store.put(START_CLOCK, getStartClock) } @@ -329,10 +337,11 @@ object ClockService { case object HealthCheck class ProcessorClock(val processorId: ProcessorId, val life: LifeTime, val parallism: Int, - private var _min: TimeStamp = 0L, private var _taskClocks: Array[TimeStamp] = null) { - + private var _min: TimeStamp = 0L, private var _taskClocks: Array[TimeStamp] = null) { - def copy(life: LifeTime): ProcessorClock = new ProcessorClock(processorId, life, parallism, _min, _taskClocks) + def copy(life: LifeTime): ProcessorClock = { + new ProcessorClock(processorId, life, parallism, _min, _taskClocks) + } def min: TimeStamp = _min def taskClocks: Array[TimeStamp] = _taskClocks @@ -362,17 +371,22 @@ object ClockService { private val LOG: Logger = LogUtil.getLogger(getClass) private var minClock: ClockValue = null - private val stallingThresholdMilliseconds = stallingThresholdSeconds * 1000 // 60 seconds + private val stallingThresholdMilliseconds = stallingThresholdSeconds * 1000 + // 60 seconds private var stallingTasks = Array.empty[TaskId] - def check(currentMinClock: TimeStamp, processorClocks: Map[ProcessorId, ProcessorClock], dag: DAG, now: TimeStamp): Unit = { + /** Check for stalling tasks */ + def check( + currentMinClock: TimeStamp, processorClocks: Map[ProcessorId, ProcessorClock], + dag: DAG, now: TimeStamp): Unit = { var isClockStalling = false if (null == minClock || currentMinClock > minClock.appClock) { minClock = ClockValue(systemClock = now, appClock = currentMinClock) } else { - //clock not advancing + // Clock not advancing if (now > minClock.systemClock + stallingThresholdMilliseconds) { - LOG.warn(s"Clock has not advanced for ${(now - minClock.systemClock)/1000} seconds since ${minClock.prettyPrint}...") + LOG.warn(s"Clock has not advanced for ${(now - minClock.systemClock) / 1000} seconds " + + s"since ${minClock.prettyPrint}...") isClockStalling = true } } @@ -387,7 +401,7 @@ object ClockService { } } - processorId.foreach {processorId => + processorId.foreach { processorId => val processorClock = processorClocks(processorId) val taskClocks = processorClock.taskClocks stallingTasks = taskClocks.zipWithIndex.filter(_._1 == minClock.appClock). @@ -414,10 +428,11 @@ object ClockService { object ProcessorClocks { + // Get the Min clock of all processors def minClock(clock: Array[ProcessorClock]): TimeStamp = { var i = 0 var min = if (clock.length == 0) 0L else clock(0).min - while(i < clock.length) { + while (i < clock.length) { min = Math.min(min, clock(i).min) i += 1 } diff --git a/streaming/src/main/scala/io/gearpump/streaming/appmaster/DagManager.scala b/streaming/src/main/scala/io/gearpump/streaming/appmaster/DagManager.scala index 7244f9595..f18b3876f 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/appmaster/DagManager.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/appmaster/DagManager.scala @@ -18,44 +18,48 @@ package io.gearpump.streaming.appmaster -import akka.actor.{ActorRef, Actor, Stash} -import io.gearpump.streaming._ -import io.gearpump.streaming.task.Subscriber -import io.gearpump.streaming.storage.AppDataStore -import io.gearpump.cluster.UserConfig -import io.gearpump.partitioner.PartitionerDescription -import DagManager._ -import io.gearpump.util.{LogUtil, Graph} +import scala.concurrent.Future + +import akka.actor.{Actor, ActorRef, Stash} import org.slf4j.Logger -import scala.concurrent.Future +import io.gearpump.cluster.UserConfig +import io.gearpump.partitioner.PartitionerDescription +import io.gearpump.streaming._ +import io.gearpump.streaming.appmaster.DagManager._ +import io.gearpump.streaming.storage.AppDataStore +import io.gearpump.streaming.task.Subscriber +import io.gearpump.util.{Graph, LogUtil} /** - * Will handle dag modification and other stuff related with DAG - */ - + * Handles dag modification and other stuff related with DAG + * + * DagManager maintains multiple version of DAGs. For each version, the DAG is immutable. + * For operations like modifying a processor, it will create a new version of DAG. + */ class DagManager(appId: Int, userConfig: UserConfig, store: AppDataStore, dag: Option[DAG]) - extends Actor with Stash { + extends Actor with Stash { + import context.dispatcher private val LOG: Logger = LogUtil.getLogger(getClass, app = appId) private val NOT_INITIALIZED = -1 private var dags = List.empty[DAG] private var maxProcessorId = -1 - implicit val system = context.system + private implicit val system = context.system private var watchers = List.empty[ActorRef] override def receive: Receive = null - override def preStart() : Unit = { + override def preStart(): Unit = { LOG.info("Initializing Dag Service, get stored Dag ....") store.get(StreamApplication.DAG).asInstanceOf[Future[DAG]].map { storedDag => if (storedDag != null) { dags :+= storedDag } else { dags :+= dag.getOrElse(DAG(userConfig.getValue[Graph[ProcessorDescription, - PartitionerDescription]](StreamApplication.DAG).get)) + PartitionerDescription]](StreamApplication.DAG).get)) } maxProcessorId = { val keys = dags.head.processors.keys @@ -91,20 +95,25 @@ class DagManager(appId: Int, userConfig: UserConfig, store: AppDataStore, dag: O def dagService: Receive = { case GetLatestDAG => + // Get the latest version of DAG. sender ! LatestDAG(dags.last) case GetTaskLaunchData(version, processorId, context) => + // Task information like Processor class, downstream subscriber processors and etc. dags.find(_.version == version).foreach { dag => LOG.info(s"Get task launcher data for processor: $processorId, dagVersion: $version") sender ! taskLaunchData(dag, processorId, context) } case ReplaceProcessor(oldProcessorId, inputNewProcessor) => + // Replace a processor with new implementation. The upstream processors and downstream + // processors are NOT changed. var newProcessor = inputNewProcessor.copy(id = nextProcessorId) - if (inputNewProcessor.jar == null){ + if (inputNewProcessor.jar == null) { val oldJar = dags.last.processors.get(oldProcessorId).get newProcessor = newProcessor.copy(jar = oldJar.jar) } if (dags.length > 1) { - sender ! DAGOperationFailed("We are in the process of handling previous dynamic dag change") + sender ! DAGOperationFailed( + "We are in the process of handling previous dynamic dag change") } else { val oldDAG = dags.last val newVersion = oldDAG.version + 1 @@ -118,21 +127,25 @@ class DagManager(appId: Int, userConfig: UserConfig, store: AppDataStore, dag: O } case WatchChange(watcher) => + // Checks whether there are modifications for this DAG. if (!this.watchers.contains(watcher)) { this.watchers :+= watcher } case NewDAGDeployed(dagVersion) => - // means the new DAG version has been successfully deployed. - // remove obsolete versions. + // Means dynamic Dag transition completed, and the new DAG version has been successfully + // deployed. The obsolete dag versions will be removed. if (dagVersion != NOT_INITIALIZED) { dags = dags.filter(_.version == dagVersion) store.put(StreamApplication.DAG, dags.last) } } - private def replaceDAG(dag: DAG, oldProcessorId: ProcessorId, newProcessor: ProcessorDescription, newVersion: Int): DAG = { - val oldProcessorLife = LifeTime(dag.processors(oldProcessorId).life.birth, newProcessor.life.birth) + private def replaceDAG( + dag: DAG, oldProcessorId: ProcessorId, newProcessor: ProcessorDescription, newVersion: Int) + : DAG = { + val oldProcessorLife = LifeTime(dag.processors(oldProcessorId).life.birth, + newProcessor.life.birth) val newProcessorMap = dag.processors ++ Map(oldProcessorId -> dag.processors(oldProcessorId).copy(life = oldProcessorLife), @@ -153,11 +166,13 @@ object DagManager { case class LatestDAG(dag: DAG) case class GetTaskLaunchData(dagVersion: Int, processorId: Int, context: AnyRef = null) - case class TaskLaunchData(processorDescription : ProcessorDescription, subscribers: List[Subscriber], context: AnyRef = null) + case class TaskLaunchData(processorDescription : ProcessorDescription, + subscribers: List[Subscriber], context: AnyRef = null) sealed trait DAGOperation - case class ReplaceProcessor(oldProcessorId: ProcessorId, newProcessorDescription: ProcessorDescription) extends DAGOperation + case class ReplaceProcessor(oldProcessorId: ProcessorId, + newProcessorDescription: ProcessorDescription) extends DAGOperation sealed trait DAGOperationResult case object DAGOperationSuccess extends DAGOperationResult diff --git a/streaming/src/main/scala/io/gearpump/streaming/appmaster/ExecutorManager.scala b/streaming/src/main/scala/io/gearpump/streaming/appmaster/ExecutorManager.scala index ebe6c1149..ab99af5d0 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/appmaster/ExecutorManager.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/appmaster/ExecutorManager.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,34 +18,32 @@ package io.gearpump.streaming.appmaster +import scala.concurrent.duration._ +import scala.util.{Failure, Try} + import akka.actor.SupervisorStrategy.Stop import akka.actor._ import akka.remote.RemoteScope import com.typesafe.config.Config -import io.gearpump.WorkerId +import org.apache.commons.lang.exception.ExceptionUtils + import io.gearpump.cluster.AppMasterToWorker.ChangeExecutorResource import io.gearpump.cluster.appmaster.ExecutorSystemScheduler.{ExecutorSystemJvmConfig, ExecutorSystemStarted, StartExecutorSystemTimeout, StartExecutorSystems} import io.gearpump.cluster.appmaster.WorkerInfo import io.gearpump.cluster.scheduler.{Resource, ResourceRequest} +import io.gearpump.cluster.worker.WorkerId import io.gearpump.cluster.{AppJar, AppMasterContext, ExecutorContext, UserConfig} import io.gearpump.streaming.ExecutorId import io.gearpump.streaming.ExecutorToAppMaster.RegisterExecutor import io.gearpump.streaming.appmaster.ExecutorManager._ import io.gearpump.streaming.executor.Executor import io.gearpump.util.{LogUtil, Util} -import org.apache.commons.lang.exception.ExceptionUtils - -import scala.concurrent.duration._ -import scala.util.Try /** - * ExecutorManager manage all executors. + * ExecutorManager manage the start and stop of all executors. * * ExecutorManager will launch Executor when asked. It hide the details like starting - * a new ExecutorSystem from user. - * - * Please use ExecutorManager.props() to construct this actor - * + * a new ExecutorSystem from user. Please use ExecutorManager.props() to construct this actor */ private[appmaster] class ExecutorManager( userConfig: UserConfig, @@ -60,10 +58,11 @@ private[appmaster] class ExecutorManager( import appContext.{appId, masterProxy, username} private var taskManager: ActorRef = null - implicit val actorSystem = context.system + + private implicit val actorSystem = context.system private val systemConfig = context.system.settings.config - private var executors = Map.empty[Int, ExecutorInfo] + private var executors = Map.empty[Int, ExecutorInfo] def receive: Receive = waitForTaskManager @@ -73,18 +72,26 @@ private[appmaster] class ExecutorManager( context.become(service orElse terminationWatch) } - override val supervisorStrategy = OneForOneStrategy(maxNrOfRetries = 10, withinTimeRange = 1 minute) { + // If something wrong on executor, ExecutorManager will stop the current executor, + // and wait for AppMaster to start a new executor. + override val supervisorStrategy = OneForOneStrategy(maxNrOfRetries = 10, + withinTimeRange = 1.minute) { case ex: Throwable => val executorId = Try(sender.path.name.toInt) executorId match { case scala.util.Success(id) => { executors -= id - LOG.error(s"Executor $id throws exception, stop it...\n" + ExceptionUtils.getStackTrace(ex)) + LOG.error(s"Executor $id throws exception, stop it...\n" + + ExceptionUtils.getStackTrace(ex)) + } + case Failure(ex) => { + LOG.error(s"Sender ${sender.path} is dead, but seems it is not an executor...") } } Stop } + // Responds to outside queries def service: Receive = { case StartExecutors(resources, jar) => masterProxy ! StartExecutorSystems(resources, getExecutorJvmConfig(Some(jar))) @@ -92,30 +99,31 @@ private[appmaster] class ExecutorManager( import executorSystem.{address, executorSystemId, resource => executorResource, worker} val executorId = executorSystemId - val executorContext = ExecutorContext(executorId, worker, appId, appName, appMaster = context.parent, executorResource) + val executorContext = ExecutorContext(executorId, worker, appId, appName, + appMaster = context.parent, executorResource) executors += executorId -> ExecutorInfo(executorId, null, worker, boundedJar) - //start executor - val executor = context.actorOf(executorFactory(executorContext, userConfig, address, executorId), - executorId.toString) + // Starts executor + val executor = context.actorOf(executorFactory(executorContext, userConfig, + address, executorId), executorId.toString) executorSystem.bindLifeCycleWith(executor) case StartExecutorSystemTimeout => taskManager ! StartExecutorsTimeOut case RegisterExecutor(executor, executorId, resource, worker) => LOG.info(s"executor $executorId has been launched") - //watch for executor termination + // Watches for executor termination context.watch(executor) val executorInfo = executors.get(executorId).get executors += executorId -> executorInfo.copy(executor = executor) taskManager ! ExecutorStarted(executorId, resource, worker.workerId, executorInfo.boundedJar) - // broadcast message to all executors + // Broadcasts message to all executors case BroadCast(msg) => LOG.info(s"Broadcast ${msg.getClass.getSimpleName} to all executors") - context.children.foreach(_ forward msg) + context.children.foreach(_ forward msg) - // unicast message to single executor + // Unicasts message to single executor case UniCast(executorId, msg) => LOG.info(s"Unicast ${msg.getClass.getSimpleName} to executor $executorId") val executor = executors.get(executorId) @@ -124,13 +132,14 @@ private[appmaster] class ExecutorManager( case GetExecutorInfo => sender ! executors - // update resource usage, so that reclaim unused resource. + // Tells Executor manager resources that are occupied. The Executor Manager can use this + // information to tell worker to reclaim un-used resources case ExecutorResourceUsageSummary(resources) => executors.foreach { pair => val (executorId, executor) = pair val resource = resources.get(executorId) val worker = executor.worker.ref - // notify the worker the actual resource used by this application. + // Notifies the worker the actual resource used by this application. resource match { case Some(resource) => worker ! ChangeExecutorResource(appId, executorId, resource) @@ -138,9 +147,9 @@ private[appmaster] class ExecutorManager( worker ! ChangeExecutorResource(appId, executorId, Resource(0)) } } - } + } - def terminationWatch : Receive = { + def terminationWatch: Receive = { case Terminated(actor) => val executorId = Try(actor.path.name.toInt) executorId match { @@ -149,14 +158,17 @@ private[appmaster] class ExecutorManager( LOG.error(s"Executor $id is down") taskManager ! ExecutorStopped(id) } - case scala.util.Failure(ex) => LOG.error(s"failed to get the executor Id from path string ${actor.path}" , ex) + case scala.util.Failure(ex) => + LOG.error(s"failed to get the executor Id from path string ${actor.path}", ex) } } private def getExecutorJvmConfig(jar: Option[AppJar]): ExecutorSystemJvmConfig = { val executorAkkaConfig = clusterConfig val jvmSetting = Util.resolveJvmSetting(executorAkkaConfig.withFallback(systemConfig)).executor - ExecutorSystemJvmConfig(jvmSetting.classPath, jvmSetting.vmargs, jar, username, executorAkkaConfig) + + ExecutorSystemJvmConfig(jvmSetting.classPath, jvmSetting.vmargs, jar, + username, executorAkkaConfig) } } @@ -168,26 +180,30 @@ private[appmaster] object ExecutorManager { case object GetExecutorInfo - case class ExecutorStarted(executorId: Int, resource: Resource, workerId: WorkerId, boundedJar: Option[AppJar]) + case class ExecutorStarted( + executorId: Int, resource: Resource, workerId: WorkerId, boundedJar: Option[AppJar]) case class ExecutorStopped(executorId: Int) case class SetTaskManager(taskManager: ActorRef) case object StartExecutorsTimeOut - def props(userConfig: UserConfig, appContext: AppMasterContext, clusterConfig: Config, appName: String): Props = { + def props( + userConfig: UserConfig, appContext: AppMasterContext, clusterConfig: Config, appName: String) + : Props = { val executorFactory = - (executorContext: ExecutorContext, - userConfig: UserConfig, - address: Address, - executorId: ExecutorId) => - Props(classOf[Executor], executorContext, userConfig) - .withDeploy(Deploy(scope = RemoteScope(address))) + (executorContext: ExecutorContext, + userConfig: UserConfig, + address: Address, + executorId: ExecutorId) => + Props(classOf[Executor], executorContext, userConfig) + .withDeploy(Deploy(scope = RemoteScope(address))) Props(new ExecutorManager(userConfig, appContext, executorFactory, clusterConfig, appName)) } case class ExecutorResourceUsageSummary(resources: Map[ExecutorId, Resource]) - case class ExecutorInfo(executorId: ExecutorId, executor: ActorRef, worker: WorkerInfo, boundedJar: Option[AppJar]) + case class ExecutorInfo( + executorId: ExecutorId, executor: ActorRef, worker: WorkerInfo, boundedJar: Option[AppJar]) } diff --git a/streaming/src/main/scala/io/gearpump/streaming/appmaster/JarScheduler.scala b/streaming/src/main/scala/io/gearpump/streaming/appmaster/JarScheduler.scala index e769a0d96..a2a9c7495 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/appmaster/JarScheduler.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/appmaster/JarScheduler.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,51 +17,69 @@ */ package io.gearpump.streaming.appmaster +import scala.concurrent.Future + import akka.actor._ +import akka.pattern.ask import com.typesafe.config.Config -import io.gearpump.{WorkerId, TimeStamp} -import io.gearpump.streaming.task.{StartClock, TaskId} -import io.gearpump.streaming.{ProcessorDescription, DAG} + +import io.gearpump.TimeStamp import io.gearpump.cluster.AppJar import io.gearpump.cluster.scheduler.{Resource, ResourceRequest} +import io.gearpump.cluster.worker.WorkerId import io.gearpump.partitioner.PartitionerDescription import io.gearpump.streaming.appmaster.JarScheduler._ -import io.gearpump.util.{LogUtil, Constants, Graph} -import akka.pattern.ask -import scala.concurrent.Future +import io.gearpump.streaming.task.TaskId +import io.gearpump.streaming.{DAG, ProcessorDescription} +import io.gearpump.util.{Constants, Graph, LogUtil} /** + * Different processors of the stream application can use different jars. JarScheduler is the + * scheduler for different jars. * - * With JarScheduler, we allows a DAG to be partitioned into several - * parts, with each part use its own jar file. + * For a DAG of multiple processors, each processor can have its own jar. Tasks of same jar + * is scheduled by TaskScheduler, and TaskSchedulers are scheduled by JarScheduler. * + * In runtime, the implementation is delegated to actor JarSchedulerImpl */ -class JarScheduler(appId : Int, appName: String, config: Config, factory: ActorRefFactory) { +class JarScheduler(appId: Int, appName: String, config: Config, factory: ActorRefFactory) { private val actor: ActorRef = factory.actorOf(Props(new JarSchedulerImpl(appId, appName, config))) private implicit val dispatcher = factory.dispatcher private implicit val timeout = Constants.FUTURE_TIMEOUT + /** Set the current DAG version active */ def setDag(dag: DAG, startClock: Future[TimeStamp]): Unit = { actor ! TransitToNewDag - startClock.map {start => + startClock.map { start => actor ! NewDag(dag, start) } } - def getRequestDetails(): Future[Array[ResourceRequestDetail]] = { - (actor ? GetRequestDetails).asInstanceOf[Future[Array[ResourceRequestDetail]]] + /** AppMaster ask JarScheduler about how many resource it wants */ + def getResourceRequestDetails(): Future[Array[ResourceRequestDetail]] = { + (actor ? GetResourceRequestDetails).asInstanceOf[Future[Array[ResourceRequestDetail]]] } - def scheduleTask(appJar: AppJar, workerId: WorkerId, executorId: Int, resource: Resource): Future[List[TaskId]] = { - (actor ? ScheduleTask(appJar, workerId, executorId, resource)).asInstanceOf[Future[List[TaskId]]] + /** + * AppMaster has resource allocated, and ask the jar scheduler to schedule tasks + * for this executor. + */ + def scheduleTask(appJar: AppJar, workerId: WorkerId, executorId: Int, resource: Resource) + : Future[List[TaskId]] = { + (actor ? ScheduleTask(appJar, workerId, executorId, resource)) + .asInstanceOf[Future[List[TaskId]]] } + /** + * Some executor JVM process is dead. AppMaster asks jar scheduler to re-schedule the impacted + * tasks. + */ def executorFailed(executorId: Int): Future[Option[ResourceRequestDetail]] = { (actor ? ExecutorFailed(executorId)).asInstanceOf[Future[Option[ResourceRequestDetail]]] } } -object JarScheduler{ +object JarScheduler { case class ResourceRequestDetail(jar: AppJar, requests: Array[ResourceRequest]) @@ -69,14 +87,24 @@ object JarScheduler{ case object TransitToNewDag - case object GetRequestDetails + case object GetResourceRequestDetails + /** + * Schedule tasks for one appJar. + * + * @param appJar Application jar. + * @param workerId Worker machine Id. + * @param executorId Executor Id. + * @param resource Slots that are available. + */ case class ScheduleTask(appJar: AppJar, workerId: WorkerId, executorId: Int, resource: Resource) + /** Some executor JVM is dead, try to recover tasks that are located on failed executor */ case class ExecutorFailed(executorId: Int) - class JarSchedulerImpl(appId : Int, appName: String, config: Config) extends Actor with Stash { + class JarSchedulerImpl(appId: Int, appName: String, config: Config) extends Actor with Stash { + // Each TaskScheduler maps to a jar. private var taskSchedulers = Map.empty[AppJar, TaskScheduler] private val LOG = LogUtil.getLogger(getClass) @@ -84,26 +112,29 @@ object JarScheduler{ def receive: Receive = waitForNewDag def waitForNewDag: Receive = { - case TransitToNewDag => // continue current state + case TransitToNewDag => // Continue current state case NewDag(dag, startTime) => LOG.info(s"Init JarScheduler, dag version: ${dag.version}, startTime: $startTime") val processors = dag.processors.values.groupBy(_.jar) + taskSchedulers = processors.map { jarAndProcessors => val (jar, processors) = jarAndProcessors - //Construct the sub DAG - val graph = Graph.empty[ProcessorDescription, PartitionerDescription] - processors.foreach{processor => + // Construct the sub DAG, each sub DAG maps to a separate jar. + val subGraph = Graph.empty[ProcessorDescription, PartitionerDescription] + processors.foreach { processor => if (startTime < processor.life.death) { - graph.addVertex(processor) + subGraph.addVertex(processor) } } - val subDag = DAG(graph) - val taskScheduler = taskSchedulers.getOrElse(jar, new TaskSchedulerImpl(appId, appName, config)) + val subDagForSingleJar = DAG(subGraph) - LOG.info(s"Set DAG for TaskScheduler, count: " + subDag.processors.size) - taskScheduler.setDAG(subDag) + val taskScheduler = taskSchedulers + .getOrElse(jar, new TaskSchedulerImpl(appId, appName, config)) + + LOG.info(s"Set DAG for TaskScheduler, count: " + subDagForSingleJar.processors.size) + taskScheduler.setDAG(subDagForSingleJar) jar -> taskScheduler } unstashAll() @@ -113,15 +144,20 @@ object JarScheduler{ } def ready: Receive = { + // Notifies there is a new DAG coming. case TransitToNewDag => context.become(waitForNewDag) - case GetRequestDetails => + + case GetResourceRequestDetails => + + // Asks each TaskScheduler (Each for one jar) the resource requests. val result: Array[ResourceRequestDetail] = taskSchedulers.map { jarAndScheduler => val (jar, scheduler) = jarAndScheduler ResourceRequestDetail(jar, scheduler.getResourceRequests()) }.toArray LOG.info(s"GetRequestDetails " + result.mkString(";")) sender ! result + case ScheduleTask(appJar, workerId, executorId, resource) => val result: List[TaskId] = taskSchedulers.get(appJar).map { scheduler => scheduler.schedule(workerId, executorId, resource) @@ -130,9 +166,9 @@ object JarScheduler{ sender ! result case ExecutorFailed(executorId) => val result: Option[ResourceRequestDetail] = taskSchedulers. - find(_._2.scheduledTasks(executorId).nonEmpty).map{ jarAndScheduler => - ResourceRequestDetail(jarAndScheduler._1, jarAndScheduler._2.executorFailed(executorId)) - } + find(_._2.scheduledTasks(executorId).nonEmpty).map { jarAndScheduler => + ResourceRequestDetail(jarAndScheduler._1, jarAndScheduler._2.executorFailed(executorId)) + } LOG.info(s"ExecutorFailed " + result.mkString(";")) sender ! result } diff --git a/streaming/src/main/scala/io/gearpump/streaming/appmaster/StreamAppMasterSummary.scala b/streaming/src/main/scala/io/gearpump/streaming/appmaster/StreamAppMasterSummary.scala index cdcd3f652..3b2c8bf90 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/appmaster/StreamAppMasterSummary.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/appmaster/StreamAppMasterSummary.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,15 +18,16 @@ package io.gearpump.streaming.appmaster -import io.gearpump.streaming.{ExecutorId, ProcessorId, LifeTime} import io.gearpump._ import io.gearpump.cluster.AppMasterToMaster.AppMasterSummary -import io.gearpump.cluster.{UserConfig, MasterToAppMaster} import io.gearpump.cluster.MasterToAppMaster.AppMasterStatus -import AppMaster.ExecutorBrief +import io.gearpump.cluster.{MasterToAppMaster, UserConfig} +import io.gearpump.streaming.appmaster.AppMaster.ExecutorBrief +import io.gearpump.streaming.{ExecutorId, LifeTime, ProcessorId} import io.gearpump.util.Graph import io.gearpump.util.HistoryMetricsService.HistoryMetricsConfig +/** Stream application summary, used for REST API */ case class StreamAppMasterSummary( appType: String = "streaming", appId: Int, @@ -42,7 +43,7 @@ case class StreamAppMasterSummary( dag: Graph[ProcessorId, String] = null, executors: List[ExecutorBrief] = null, processors: Map[ProcessorId, ProcessorSummary] = Map.empty[ProcessorId, ProcessorSummary], - // hiearachy level for each processor + // Hiearachy level for each processor processorLevels: Map[ProcessorId, Int] = Map.empty[ProcessorId, Int], historyMetricsConfig: HistoryMetricsConfig = null) extends AppMasterSummary @@ -50,11 +51,11 @@ case class StreamAppMasterSummary( case class TaskCount(count: Int) case class ProcessorSummary( - id: ProcessorId, - taskClass: String, - parallelism : Int, - description: String, - taskConf: UserConfig, - life: LifeTime, - executors: List[ExecutorId], - taskCount: Map[ExecutorId, TaskCount]) \ No newline at end of file + id: ProcessorId, + taskClass: String, + parallelism: Int, + description: String, + taskConf: UserConfig, + life: LifeTime, + executors: List[ExecutorId], + taskCount: Map[ExecutorId, TaskCount]) \ No newline at end of file diff --git a/streaming/src/main/scala/io/gearpump/streaming/appmaster/TaskLocator.scala b/streaming/src/main/scala/io/gearpump/streaming/appmaster/TaskLocator.scala index c456a06e8..6e7ebc61f 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/appmaster/TaskLocator.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/appmaster/TaskLocator.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,35 +17,38 @@ */ package io.gearpump.streaming.appmaster -import com.typesafe.config.{ConfigValueFactory, ConfigFactory, ConfigRenderOptions, Config} -import TaskLocator.{Localities, WorkerLocality, NonLocality, Locality} -import io.gearpump.WorkerId +import scala.collection.JavaConverters._ +import scala.util.Try + +import com.typesafe.config.{Config, ConfigFactory, ConfigRenderOptions, ConfigValueFactory} + +import io.gearpump.cluster.worker.WorkerId import io.gearpump.streaming.Constants +import io.gearpump.streaming.appmaster.TaskLocator.{Localities, Locality, NonLocality, WorkerLocality} import io.gearpump.streaming.task.TaskId -import scala.util.Try -import scala.collection.JavaConverters._ /** * TaskLocator is used to decide which machine one task should run on. * - * User can specify config [[Constants.GEARPUMP_STREAMING_LOCALITIES]] to decide - * to control which machine the task is running on. + * User can specify config [[io.gearpump.streaming.Constants#GEARPUMP_STREAMING_LOCALITIES]] to + * decide to control which machine the task is running on. */ class TaskLocator(appName: String, config: Config) { private val taskLocalities: Map[TaskId, Locality] = loadTaskLocalities(config) - def locateTask(taskId: TaskId) : Locality = { + /** Finds where a task should belongs to */ + def locateTask(taskId: TaskId): Locality = { taskLocalities.getOrElse(taskId, NonLocality) } - private def loadTaskLocalities(config: Config) : Map[TaskId, Locality] = { - import Constants.GEARPUMP_STREAMING_LOCALITIES - Try(config.getConfig(s"$GEARPUMP_STREAMING_LOCALITIES.$appName")).map {appConfig => + private def loadTaskLocalities(config: Config): Map[TaskId, Locality] = { + import io.gearpump.streaming.Constants.GEARPUMP_STREAMING_LOCALITIES + Try(config.getConfig(s"$GEARPUMP_STREAMING_LOCALITIES.$appName")).map { appConfig => val json = appConfig.root().render(ConfigRenderOptions.concise) Localities.fromJson(json) }.map { localityConfig => import localityConfig.localities - localities.keySet.flatMap {workerId => + localities.keySet.flatMap { workerId => val tasks = localities(workerId) tasks.map((_, WorkerLocality(workerId))) }.toArray.toMap @@ -57,15 +60,20 @@ object TaskLocator { trait Locality + /** Means we require the resource from the specific worker */ case class WorkerLocality(workerId: WorkerId) extends Locality + /** Means no preference on worker */ object NonLocality extends Locality + /** Localities settings. Mapping from workerId to list of taskId */ case class Localities(localities: Map[WorkerId, Array[TaskId]]) object Localities { val pattern = "task_([0-9]+)_([0-9]+)".r + // To avoid polluting the classpath, we do the JSON translation ourself instead of + // introducing JSON library dependencies directly. def fromJson(json: String): Localities = { val localities = ConfigFactory.parseString(json).getAnyRef("localities") .asInstanceOf[java.util.Map[String, String]].asScala.map { pair => @@ -80,8 +88,9 @@ object TaskLocator { } def toJson(localities: Localities): String = { - val map = localities.localities.toList.map {pair => - (WorkerId.render(pair._1), pair._2.map(task => s"task_${task.processorId}_${task.index}").mkString(",")) + val map = localities.localities.toList.map { pair => + (WorkerId.render(pair._1), pair._2.map(task => + s"task_${task.processorId}_${task.index}").mkString(",")) }.toMap.asJava ConfigFactory.empty().withValue("localities", ConfigValueFactory.fromAnyRef(map)). root.render(ConfigRenderOptions.concise()) diff --git a/streaming/src/main/scala/io/gearpump/streaming/appmaster/TaskManager.scala b/streaming/src/main/scala/io/gearpump/streaming/appmaster/TaskManager.scala index e1f4872c1..51f2f9c61 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/appmaster/TaskManager.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/appmaster/TaskManager.scala @@ -18,12 +18,17 @@ package io.gearpump.streaming.appmaster +import scala.concurrent.Future +import scala.concurrent.duration._ + import akka.actor._ import akka.pattern.ask +import org.slf4j.Logger + import io.gearpump.TimeStamp import io.gearpump.cluster.MasterToAppMaster.ReplayFromTimestampWindowTrailingEdge import io.gearpump.streaming.AppMasterToExecutor._ -import io.gearpump.streaming.ExecutorToAppMaster.{UnRegisterTask, MessageLoss, RegisterTask} +import io.gearpump.streaming.ExecutorToAppMaster.{MessageLoss, RegisterTask, UnRegisterTask} import io.gearpump.streaming._ import io.gearpump.streaming.appmaster.AppMaster.{AllocateResourceTimeOut, LookupTaskActorRef, TaskActorRef} import io.gearpump.streaming.appmaster.ClockService.{ChangeToNewDAG, ChangeToNewDAGSuccess} @@ -36,31 +41,26 @@ import io.gearpump.streaming.executor.ExecutorRestartPolicy import io.gearpump.streaming.task._ import io.gearpump.streaming.util.ActorPathUtil import io.gearpump.util.{Constants, LogUtil} -import org.slf4j.Logger - -import scala.concurrent.Future -import scala.concurrent.duration._ /** * * TaskManager track all tasks's status. * * It is state machine with three states: - * 1. applicationReady - * 2. recovery - * 3. dynamicDag + * 1. applicationReady + * 2. recovery + * 3. dynamicDag * * When in state applicationReady: - * 1. When there is message-loss or JVM crash, transit to state recovery. - * 2. When user modify the DAG, transit to dynamicDag. + * 1. When there is message-loss or JVM crash, transit to state recovery. + * 2. When user modify the DAG, transit to dynamicDag. * * When in state recovery: - * 1. When all tasks has been recovered, transit to applicationReady. + * 1. When all tasks has been recovered, transit to applicationReady. * * When in state dynamicDag: - * 1. When dyanmic dag transition is complete, transit to applicationReady. - * 2. When there is message loss or JVM crash, transit to state recovery. - * + * 1. When dynamic dag transition is complete, transit to applicationReady. + * 2. When there is message loss or JVM crash, transit to state recovery. */ private[appmaster] class TaskManager( appId: Int, @@ -73,13 +73,16 @@ private[appmaster] class TaskManager( extends Actor { private val LOG: Logger = LogUtil.getLogger(getClass, app = appId) - val systemConfig = context.system.settings.config + private val systemConfig = context.system.settings.config private val ids = new SessionIdFactory() - private val executorRestartPolicy = new ExecutorRestartPolicy(maxNrOfRetries = 5, withinTimeRange = 20 seconds) - implicit val timeout = Constants.FUTURE_TIMEOUT - implicit val actorSystem = context.system + private val executorRestartPolicy = new ExecutorRestartPolicy(maxNrOfRetries = 5, + withinTimeRange = 20.seconds) + + private implicit val timeout = Constants.FUTURE_TIMEOUT + private implicit val actorSystem = context.system + import context.dispatcher dagManager ! WatchChange(watcher = self) @@ -91,7 +94,6 @@ private[appmaster] class TaskManager( private var startClock: Future[TimeStamp] = getStartClock - def receive: Receive = applicationReady(DagReadyState.empty) private def onClientQuery(taskRegistry: TaskRegistry): Receive = { @@ -104,14 +106,14 @@ private[appmaster] class TaskManager( val requestor = sender executorId.map { executorId => val taskPath = ActorPathUtil.taskActorPath(appMaster, executorId, taskId) - context.actorSelection(taskPath).resolveOne(3 seconds).map { taskActorRef => + context.actorSelection(taskPath).resolveOne(3.seconds).map { taskActorRef => requestor ! TaskActorRef(taskActorRef) } } } /** - * state applicationReady + * State applicationReady */ def applicationReady(state: DagReadyState): Receive = { executorManager ! state.taskRegistry.usedResource @@ -121,7 +123,7 @@ private[appmaster] class TaskManager( val recoverRegistry = new TaskRegistry(expectedTasks = state.dag.tasks, deadTasks = state.taskRegistry.deadTasks) - + val recoverState = new StartDagState(state.dag, recoverRegistry) val onError: Receive = { @@ -153,8 +155,8 @@ private[appmaster] class TaskManager( val dagDiff = migrate(state.dag, newDag) jarScheduler.setDag(newDag, startClock) - val resourceRequestsDetails = jarScheduler.getRequestDetails() - resourceRequestsDetails.map{ details => + val resourceRequestsDetails = jarScheduler.getResourceRequestDetails() + resourceRequestsDetails.map { details => details.foreach { detail => if (detail.requests.length > 0 && detail.requests.exists(!_.resource.isEmpty)) { executorManager ! StartExecutors(detail.requests, detail.jar) @@ -168,7 +170,8 @@ private[appmaster] class TaskManager( executors.foreach { pair => val (executorId, tasks) = pair modifiedTasks ++= tasks - dagManager ! GetTaskLaunchData(newDag.version, processorId, ChangeTasksOnExecutor(executorId, tasks)) + dagManager ! GetTaskLaunchData(newDag.version, processorId, + ChangeTasksOnExecutor(executorId, tasks)) } } @@ -200,12 +203,13 @@ private[appmaster] class TaskManager( context.become(applicationReady(newState)) } - // recover to same version - onClientQuery(state.taskRegistry) orElse onError orElse onNewDag orElse onUnRegisterTask orElse unHandled("applicationReady") + // Recovers to same version + onClientQuery(state.taskRegistry) orElse onError orElse onNewDag orElse + onUnRegisterTask orElse unHandled("applicationReady") } /** - * state dynamicDag + * State dynamicDag */ def dynamicDag(state: StartDagState, recoverState: StartDagState): Receive = { LOG.info(s"DynamicDag transit to dag version: ${state.dag.version}...") @@ -231,11 +235,13 @@ private[appmaster] class TaskManager( case executor: ExecutorStarted => import executor.{boundedJar, executorId, resource, workerId} val taskIdsFuture = jarScheduler.scheduleTask(boundedJar.get, workerId, executorId, resource) - taskIdsFuture.foreach {taskIds => - LOG.info(s"Executor $executor has been started, start to schedule tasks: ${taskIds.mkString(",")}") + taskIdsFuture.foreach { taskIds => + LOG.info(s"Executor $executor has been started, " + + s"start to schedule tasks: ${taskIds.mkString(",")}") taskIds.groupBy(_.processorId).foreach { pair => val (processorId, tasks) = pair - dagManager ! GetTaskLaunchData(state.dag.version, processorId, StartTasksOnExecutor(executor.executorId, tasks)) + dagManager ! GetTaskLaunchData(state.dag.version, processorId, + StartTasksOnExecutor(executor.executorId, tasks)) } } @@ -250,7 +256,8 @@ private[appmaster] class TaskManager( tasks.foreach(executorRestartPolicy.addTaskToExecutor(executorId, _)) case ChangeTasksOnExecutor(executorId, tasks) => LOG.info("change Task on executor: " + executorId + ", tasks: " + tasks) - val changeTasks = ChangeTasks(tasks, state.dag.version, processorDescription.life, subscribers) + val changeTasks = ChangeTasks(tasks, state.dag.version, processorDescription.life, + subscribers) executorManager ! UniCast(executorId, changeTasks) case other => LOG.error(s"severe error! we expect ExecutorStarted but get ${other.getClass.toString}") @@ -258,7 +265,7 @@ private[appmaster] class TaskManager( case TasksLaunched => // We will track all launched task by message RegisterTask case TasksChanged(tasks) => - tasks.foreach(task =>state.taskChangeRegistry.taskChanged(task)) + tasks.foreach(task => state.taskChangeRegistry.taskChanged(task)) if (allTasksReady(state)) { broadcastLocations(state) @@ -306,10 +313,11 @@ private[appmaster] class TaskManager( def onExecutorError: Receive = { case ExecutorStopped(executorId) => - if(executorRestartPolicy.allowRestartExecutor(executorId)) { + if (executorRestartPolicy.allowRestartExecutor(executorId)) { jarScheduler.executorFailed(executorId).foreach { resourceRequestDetail => if (resourceRequestDetail.isDefined) { - executorManager ! StartExecutors(resourceRequestDetail.get.requests, resourceRequestDetail.get.jar) + executorManager ! StartExecutors(resourceRequestDetail.get.requests, + resourceRequestDetail.get.jar) } } } else { @@ -330,7 +338,7 @@ private[appmaster] class TaskManager( } /** - * state recovery + * State recovery */ def recovery(state: StartDagState): Receive = { val recoverDagVersion = state.dag.version @@ -344,7 +352,7 @@ private[appmaster] class TaskManager( LOG.info(s"goto state Recovery(recoverDag = $recoverDagVersion)...") val ignoreClock: Receive = { case clock: ClockEvent => - //ignore clock events. + // Ignores clock events. } if (state.dag.isEmpty) { @@ -354,7 +362,8 @@ private[appmaster] class TaskManager( deadTasks = state.taskRegistry.deadTasks) val recoverState = new StartDagState(state.dag, registry) - ignoreClock orElse startDag(state, recoverState) orElse onExecutorError orElse unHandled("recovery") + ignoreClock orElse startDag(state, recoverState) orElse onExecutorError orElse + unHandled("recovery") } } @@ -364,19 +373,17 @@ private[appmaster] class TaskManager( } } -private [appmaster] object TaskManager { +private[appmaster] object TaskManager { /** * When application is ready, then transit to DagReadyState */ - class DagReadyState( - val dag: DAG, - val taskRegistry: TaskRegistry) + class DagReadyState(val dag: DAG, val taskRegistry: TaskRegistry) object DagReadyState { def empty: DagReadyState = { new DagReadyState( - DAG.empty().copy(version = -1), + DAG.empty.copy(version = -1), new TaskRegistry(List.empty[TaskId])) } } @@ -385,10 +392,10 @@ private [appmaster] object TaskManager { * When application is booting up or doing recovery, it use StartDagState */ class StartDagState( - val dag: DAG, - val taskRegistry: TaskRegistry, - val taskChangeRegistry: TaskChangeRegistry = new TaskChangeRegistry(List.empty[TaskId]), - val executorReadyRegistry: ExecutorRegistry = new ExecutorRegistry) + val dag: DAG, + val taskRegistry: TaskRegistry, + val taskChangeRegistry: TaskChangeRegistry = new TaskChangeRegistry(List.empty[TaskId]), + val executorReadyRegistry: ExecutorRegistry = new ExecutorRegistry) case object GetTaskList @@ -397,18 +404,17 @@ private [appmaster] object TaskManager { case class FailedToRecover(errorMsg: String) /** - * Start new Tasks on Executor + * Starts new Tasks on Executor executorId */ case class StartTasksOnExecutor(executorId: Int, tasks: List[TaskId]) /** - * Change existing tasks on executor + * Changes existing tasks on executor executorId */ case class ChangeTasksOnExecutor(executorId: Int, tasks: List[TaskId]) - /** - * Track the registration of all new started executors. + * Tracks the registration of all new started executors. */ class ExecutorRegistry { private var registeredExecutors = Set.empty[ExecutorId] @@ -423,7 +429,7 @@ private [appmaster] object TaskManager { } /** - * Track the registration of all changed tasks. + * Tracks the registration of all changed tasks. */ class TaskChangeRegistry(targetTasks: List[TaskId]) { private var registeredTasks = Set.empty[TaskId] @@ -443,12 +449,12 @@ private [appmaster] object TaskManager { * DAGDiff is used to track impacted processors when doing dynamic dag. */ case class DAGDiff( - addedProcessors: List[ProcessorId], - modifiedProcessors: List[ProcessorId], - impactedUpstream: List[ProcessorId]) + addedProcessors: List[ProcessorId], + modifiedProcessors: List[ProcessorId], + impactedUpstream: List[ProcessorId]) /** - * Migrate from old DAG to new DAG, return DAGDiff + * Migrates from old DAG to new DAG, return DAGDiff */ def migrate(leftDAG: DAG, rightDAG: DAG): DAGDiff = { val left = leftDAG.processors.keySet @@ -457,19 +463,19 @@ private [appmaster] object TaskManager { val added = right -- left val join = right -- added - val modified = join.filter {processorId => + val modified = join.filter { processorId => leftDAG.processors(processorId) != rightDAG.processors(processorId) } val upstream = (list: Set[ProcessorId]) => { - list.flatMap {processorId => + list.flatMap { processorId => rightDAG.graph.incomingEdgesOf(processorId).map(_._1).toSet } -- list } val impactedUpstream = upstream(added ++ modified) - // all upstream will be affected. + // All upstream tasks are affected, and should be handled properly. DAGDiff(added.toList, modified.toList, impactedUpstream.toList) } @@ -480,7 +486,7 @@ private [appmaster] object TaskManager { private var nextSessionId = 1 /** - * return a new session Id for new task + * Returns a new session Id for new task */ final def newSessionId: Int = { val sessionId = nextSessionId diff --git a/streaming/src/main/scala/io/gearpump/streaming/appmaster/TaskRegistry.scala b/streaming/src/main/scala/io/gearpump/streaming/appmaster/TaskRegistry.scala index 79371eac3..adfdeba60 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/appmaster/TaskRegistry.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/appmaster/TaskRegistry.scala @@ -18,18 +18,18 @@ package io.gearpump.streaming.appmaster -import io.gearpump.streaming.{ExecutorId, ProcessorId} -import io.gearpump.streaming.task.TaskId +import org.slf4j.Logger + import io.gearpump.cluster.scheduler.Resource -import ExecutorManager.ExecutorResourceUsageSummary -import TaskRegistry._ +import io.gearpump.streaming.appmaster.ExecutorManager.ExecutorResourceUsageSummary +import io.gearpump.streaming.appmaster.TaskRegistry._ +import io.gearpump.streaming.task.TaskId +import io.gearpump.streaming.{ExecutorId, ProcessorId} import io.gearpump.transport.HostPort import io.gearpump.util.LogUtil -import org.slf4j.Logger /** - * TaskRegistry is used to track the registration of all tasks - * when one application is booting up. + * Tracks the registration of all tasks, when the application is booting up. */ class TaskRegistry(val expectedTasks: List[TaskId], var registeredTasks: Map[TaskId, TaskLocation] = Map.empty[TaskId, TaskLocation], @@ -43,6 +43,10 @@ class TaskRegistry(val expectedTasks: List[TaskId], * When a task is booted, it need to call registerTask to register itself. * If this task is valid, then accept it, otherwise reject it. * + * @param taskId Task that register itself to TaskRegistry. + * @param location The host and port where this task is running on. NOTE: The host and port + * is NOT the same host and port of Akka remoting. Instead, it is host and port + * of custom netty layer, see [[io.gearpump.transport.netty.Context]]. */ def registerTask(taskId: TaskId, location: TaskLocation): RegisterTaskStatus = { val processorId = taskId.processorId @@ -57,13 +61,13 @@ class TaskRegistry(val expectedTasks: List[TaskId], } def copy(expectedTasks: List[TaskId] = this.expectedTasks, - registeredTasks: Map[TaskId, TaskLocation] = this.registeredTasks, - deadTasks: Set[TaskId] = this.deadTasks): TaskRegistry = { + registeredTasks: Map[TaskId, TaskLocation] = this.registeredTasks, + deadTasks: Set[TaskId] = this.deadTasks): TaskRegistry = { new TaskRegistry(expectedTasks, registeredTasks, deadTasks) } def getTaskLocations: TaskLocations = { - val taskLocations = registeredTasks.toList.groupBy(_._2.host).map{ pair => + val taskLocations = registeredTasks.toList.groupBy(_._2.host).map { pair => val (k, v) = pair val taskIds = v.map(_._1) (k, taskIds.toSet) @@ -74,16 +78,18 @@ class TaskRegistry(val expectedTasks: List[TaskId], def getTaskExecutorMap: Map[TaskId, ExecutorId] = { getTaskLocations.locations.flatMap { pair => val (hostPort, taskSet) = pair - taskSet.map{ taskId => + taskSet.map { taskId => (taskId, getExecutorId(taskId).getOrElse(-1)) } } } + /** Query the executor Id where the task is running on */ def getExecutorId(taskId: TaskId): Option[Int] = { registeredTasks.get(taskId).map(_.executorId) } + /** Gets list of allocated executor Ids */ def executors: List[ExecutorId] = { registeredTasks.toList.map(_._2.executorId) } @@ -101,6 +107,7 @@ class TaskRegistry(val expectedTasks: List[TaskId], registeredTasks.keys.toList.filter(_.processorId == processorId) } + /** List of executors that current processor taks are running on */ def processorExecutors(processorId: ProcessorId): Map[ExecutorId, List[TaskId]] = { val taskToExecutor = filterTasks(processorId).flatMap { taskId => getExecutorId(taskId).map { executorId => @@ -108,15 +115,16 @@ class TaskRegistry(val expectedTasks: List[TaskId], } } - val executorToTasks = taskToExecutor.groupBy(_._2).map{kv => + val executorToTasks = taskToExecutor.groupBy(_._2).map { kv => val (k, v) = kv (k, v.map(_._1)) } executorToTasks } + /** Summary about how many resources are used for all running tasks */ def usedResource: ExecutorResourceUsageSummary = { - val resourceMap = registeredTasks.foldLeft(Map.empty[ExecutorId, Resource]) {(map, task) => + val resourceMap = registeredTasks.foldLeft(Map.empty[ExecutorId, Resource]) { (map, task) => val resource = map.getOrElse(task._2.executorId, Resource(0)) + Resource(1) map + (task._2.executorId -> resource) } @@ -131,5 +139,5 @@ object TaskRegistry { case class TaskLocation(executorId: Int, host: HostPort) - case class TaskLocations(locations : Map[HostPort, Set[TaskId]]) + case class TaskLocations(locations: Map[HostPort, Set[TaskId]]) } \ No newline at end of file diff --git a/streaming/src/main/scala/io/gearpump/streaming/appmaster/TaskSchedulerImpl.scala b/streaming/src/main/scala/io/gearpump/streaming/appmaster/TaskSchedulerImpl.scala index 39f427ab1..62dff6cd6 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/appmaster/TaskSchedulerImpl.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/appmaster/TaskSchedulerImpl.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,17 +18,19 @@ package io.gearpump.streaming.appmaster import com.typesafe.config.Config -import io.gearpump.{WorkerId, TimeStamp} + import io.gearpump.cluster.scheduler.{Relaxation, Resource, ResourceRequest} +import io.gearpump.cluster.worker.WorkerId import io.gearpump.streaming.DAG import io.gearpump.streaming.appmaster.TaskLocator.{Locality, WorkerLocality} import io.gearpump.streaming.appmaster.TaskScheduler.{Location, TaskStatus} import io.gearpump.streaming.task.TaskId -import io.gearpump.util.{Constants, LogUtil} -import org.slf4j.Logger +import io.gearpump.util.Constants /** - * This schedules tasks to run for new allocated resources. + * Schedules tasks to run for new allocated resources. TaskScheduler only schedule tasks that + * share the same jar. For scheduling for multiple jars, see + * [[io.gearpump.streaming.appmaster.JarScheduler]]. */ trait TaskScheduler { @@ -44,8 +46,8 @@ trait TaskScheduler { def getResourceRequests(): Array[ResourceRequest] /** - * This notify the scheduler that a resource slot on {workerId} and {executorId} is allocated, and - * expect a task to be scheduled in return. + * This notifies the scheduler that a resource slot on {workerId} and {executorId} is allocated + * , and expect a task to be scheduled in return. * Task locality should be considered when deciding whether to offer a task on target {worker} * and {executor} * @@ -53,24 +55,24 @@ trait TaskScheduler { * @param executorId which executorId this resource belongs to. * @return a list of tasks */ - def schedule(workerId : WorkerId, executorId: Int, resource: Resource) : List[TaskId] + def schedule(workerId: WorkerId, executorId: Int, resource: Resource): List[TaskId] /** - * This notify the scheduler that {executorId} is failed, and expect a set of + * This notifies the scheduler that {executorId} is failed, and expect a set of * ResourceRequest for all failed tasks on that executor. * * @param executorId executor that failed * @return resource requests of the failed executor */ - def executorFailed(executorId: Int) : Array[ResourceRequest] + def executorFailed(executorId: Int): Array[ResourceRequest] /** - * Query the task list that already scheduled on the executor - * - * @param executorId executor to query - * @return a list of tasks - */ - def scheduledTasks(executorId: Int) : List[TaskId] + * Queries the task list that already scheduled on the executor + * + * @param executorId executor to query + * @return a list of tasks + */ + def scheduledTasks(executorId: Int): List[TaskId] } object TaskScheduler { @@ -79,12 +81,12 @@ object TaskScheduler { class TaskStatus(val taskId: TaskId, val preferLocality: Locality, var allocation: Location) } -class TaskSchedulerImpl(appId : Int, appName: String, config: Config) extends TaskScheduler { +class TaskSchedulerImpl(appId: Int, appName: String, config: Config) extends TaskScheduler { private val executorNum = config.getInt(Constants.APPLICATION_EXECUTOR_NUMBER) private var tasks = List.empty[TaskStatus] - // find the locality of the tasks + // Finds the locality of the tasks private val taskLocator = new TaskLocator(appName, config) override def setDAG(dag: DAG): Unit = { @@ -96,15 +98,15 @@ class TaskSchedulerImpl(appId : Int, appName: String, config: Config) extends T } } - def getResourceRequests(): Array[ResourceRequest] ={ + def getResourceRequests(): Array[ResourceRequest] = { fetchResourceRequests(fromOneWorker = false) } - import Relaxation._ - private def fetchResourceRequests(fromOneWorker: Boolean = false): Array[ResourceRequest] ={ + import io.gearpump.cluster.scheduler.Relaxation._ + private def fetchResourceRequests(fromOneWorker: Boolean = false): Array[ResourceRequest] = { var workersResourceRequest = Map.empty[WorkerId, Resource] - tasks.filter(_.allocation == null).foreach{task => + tasks.filter(_.allocation == null).foreach { task => task.preferLocality match { case WorkerLocality(workerId) => val current = workersResourceRequest.getOrElse(workerId, Resource.empty) @@ -116,7 +118,7 @@ class TaskSchedulerImpl(appId : Int, appName: String, config: Config) extends T } } - workersResourceRequest.map {workerIdAndResource => + workersResourceRequest.map { workerIdAndResource => val (workerId, resource) = workerIdAndResource if (workerId == WorkerId.unspecified) { ResourceRequest(resource, workerId = WorkerId.unspecified, executorNum = executorNum) @@ -126,23 +128,26 @@ class TaskSchedulerImpl(appId : Int, appName: String, config: Config) extends T }.toArray } - override def schedule(workerId : WorkerId, executorId: Int, resource: Resource) : List[TaskId] = { + override def schedule(workerId: WorkerId, executorId: Int, resource: Resource): List[TaskId] = { var scheduledTasks = List.empty[TaskId] val location = Location(workerId, executorId) - // schedule tasks for specific worker + // Schedules tasks for specific worker scheduledTasks ++= scheduleTasksForLocality(resource, location, (locality) => locality == WorkerLocality(workerId)) - // schedule tasks without specific location preference - scheduledTasks ++= scheduleTasksForLocality(resource - Resource(scheduledTasks.length), location, (locality) => true) + // Schedules tasks without specific location preference + scheduledTasks ++= scheduleTasksForLocality(resource - Resource(scheduledTasks.length), + location, (locality) => true) scheduledTasks } - private def scheduleTasksForLocality(resource: Resource, resourceLocation: Location, matcher: (Locality) => Boolean): List[TaskId] = { + private def scheduleTasksForLocality( + resource: Resource, resourceLocation: Location, matcher: (Locality) => Boolean) + : List[TaskId] = { var scheduledTasks = List.empty[TaskId] var index = 0 var remain = resource.slots - while(index < tasks.length && remain > 0) { + while (index < tasks.length && remain > 0) { val taskStatus = tasks(index) if (taskStatus.allocation == null && matcher(taskStatus.preferLocality)) { taskStatus.allocation = resourceLocation @@ -154,18 +159,19 @@ class TaskSchedulerImpl(appId : Int, appName: String, config: Config) extends T scheduledTasks } - override def executorFailed(executorId: Int) : Array[ResourceRequest] = { + override def executorFailed(executorId: Int): Array[ResourceRequest] = { val failedTasks = tasks.filter { status => status.allocation != null && status.allocation.executorId == executorId } - // clean the location of failed tasks + // Cleans the location of failed tasks failedTasks.foreach(_.allocation = null) - Array(ResourceRequest(Resource(failedTasks.length), workerId = WorkerId.unspecified, relaxation = ONEWORKER)) + Array(ResourceRequest(Resource(failedTasks.length), + workerId = WorkerId.unspecified, relaxation = ONEWORKER)) } override def scheduledTasks(executorId: Int): List[TaskId] = { - tasks.filter{ status => + tasks.filter { status => status.allocation != null && status.allocation.executorId == executorId }.map(_.taskId) } diff --git a/streaming/src/main/scala/io/gearpump/streaming/dsl/Stream.scala b/streaming/src/main/scala/io/gearpump/streaming/dsl/Stream.scala index 7fcaccfef..ddd20371f 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/dsl/Stream.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/dsl/Stream.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,55 +18,62 @@ package io.gearpump.streaming.dsl +import scala.language.implicitConversions + +import org.slf4j.{Logger, LoggerFactory} + import io.gearpump.Message import io.gearpump.cluster.UserConfig import io.gearpump.streaming.dsl.op._ import io.gearpump.streaming.sink.DataSink -import io.gearpump.streaming.task.{TaskContext, Task} -import io.gearpump.util.{Graph, LogUtil} -import org.slf4j.{Logger, LoggerFactory} +import io.gearpump.streaming.task.{Task, TaskContext} +import io.gearpump.util.Graph -class Stream[T](private val graph: Graph[Op,OpEdge], private val thisNode:Op, private val edge: Option[OpEdge] = None) { +class Stream[T]( + private val graph: Graph[Op, OpEdge], private val thisNode: Op, + private val edge: Option[OpEdge] = None) { /** - * convert a value[T] to a list of value[R] - * @param fun function - * @param description the description message for this operation - * @param the result message type - * @return a new stream with type [R] + * converts a value[T] to a list of value[R] + * + * @param fun FlatMap function + * @param description The description message for this operation + * @return A new stream with type [R] */ def flatMap[R](fun: T => TraversableOnce[R], description: String = null): Stream[R] = { val flatMapOp = FlatMapOp(fun, Option(description).getOrElse("flatmap")) - graph.addVertex(flatMapOp ) + graph.addVertex(flatMapOp) graph.addEdge(thisNode, edge.getOrElse(Direct), flatMapOp) new Stream[R](graph, flatMapOp) } /** - * convert value[T] to value[R] - * @param fun function - * @param the result message type - * @return a new stream with type [R] + * Maps message of type T message of type R + * + * @param fun Function + * @return A new stream with type [R] */ def map[R](fun: T => R, description: String = null): Stream[R] = { - this.flatMap ({ data => + this.flatMap({ data => Option(fun(data)) }, Option(description).getOrElse("map")) } /** - * reserve records when fun(T) == true + * Keeps records when fun(T) == true + * * @param fun the filter * @return a new stream after filter */ def filter(fun: T => Boolean, description: String = null): Stream[T] = { - this.flatMap ({ data => + this.flatMap({ data => if (fun(data)) Option(data) else None }, Option(description).getOrElse("filter")) } /** - * Reduce opeartion + * Reduces operations. + * * @param fun reduction function * @param description description message for this operator * @return a new stream after reduction @@ -86,7 +93,8 @@ class Stream[T](private val graph: Graph[Op,OpEdge], private val thisNode:Op, pr } /** - * Merge data from two stream into one + * Merges data from two stream into one + * * @param other the other stream * @return the merged stream */ @@ -99,7 +107,7 @@ class Stream[T](private val graph: Graph[Op,OpEdge], private val thisNode:Op, pr } /** - * Group by fun(T) + * Group by function (T => Group) * * For example, we have T type, People(name: String, gender: String, age: Int) * groupBy[People](_.gender) will group the people by gender. @@ -107,16 +115,17 @@ class Stream[T](private val graph: Graph[Op,OpEdge], private val thisNode:Op, pr * You can append other combinators after groupBy * * For example, - * + * {{{ * Stream[People].groupBy(_.gender).flatmap(..).filter.(..).reduce(..) + * }}} * - * @param fun group by function - * @param parallelism parallelism level - * @param description the description - * @param the group type + * @param fun Group by function + * @param parallelism Parallelism level + * @param description The description * @return the grouped stream */ - def groupBy[Group](fun: T => Group, parallelism: Int = 1, description: String = null): Stream[T] = { + def groupBy[Group](fun: T => Group, parallelism: Int = 1, description: String = null) + : Stream[T] = { val groupOp = GroupByOp(fun, parallelism, Option(description).getOrElse("groupBy")) graph.addVertex(groupOp) graph.addEdge(thisNode, edge.getOrElse(Shuffle), groupOp) @@ -124,33 +133,36 @@ class Stream[T](private val graph: Graph[Op,OpEdge], private val thisNode:Op, pr } /** - * connect with a low level Processor(TaskDescription) + * Connects with a low level Processor(TaskDescription) + * * @param processor a user defined processor * @param parallelism parallelism level - * @param the result message type * @return new stream after processing with type [R] */ - def process[R](processor: Class[_ <: Task], parallism: Int, conf: UserConfig = UserConfig.empty, description: String = null): Stream[R] = { - val processorOp = ProcessorOp(processor, parallism, conf, Option(description).getOrElse("process")) + def process[R]( + processor: Class[_ <: Task], parallelism: Int, conf: UserConfig = UserConfig.empty, + description: String = null): Stream[R] = { + val processorOp = ProcessorOp(processor, parallelism, conf, + Option(description).getOrElse("process")) graph.addVertex(processorOp) graph.addEdge(thisNode, edge.getOrElse(Shuffle), processorOp) new Stream[R](graph, processorOp, Some(Shuffle)) } } -class KVStream[K, V](stream: Stream[Tuple2[K, V]]){ +class KVStream[K, V](stream: Stream[Tuple2[K, V]]) { /** - * Apply to Stream[Tuple2[K,V]] - * Group by the key of a KV tuple - * For (key, value) will groupby key + * GroupBy key + * + * Applies to Stream[Tuple2[K,V]] + * * @param parallelism the parallelism for this operation * @return the new KV stream */ - def groupByKey(parallism: Int = 1): Stream[Tuple2[K, V]] = { - stream.groupBy(Stream.getTupleKey[K, V], parallism, "groupByKey") + def groupByKey(parallelism: Int = 1): Stream[Tuple2[K, V]] = { + stream.groupBy(Stream.getTupleKey[K, V], parallelism, "groupByKey") } - /** * Sum the value of the tuples * @@ -160,31 +172,39 @@ class KVStream[K, V](stream: Stream[Tuple2[K, V]]){ * @param numeric the numeric operations * @return the sum stream */ - def sum(implicit numeric: Numeric[V]) = { + def sum(implicit numeric: Numeric[V]): Stream[(K, V)] = { stream.reduce(Stream.sumByValue[K, V](numeric), "sum") } } object Stream { - def apply[T](graph: Graph[Op, OpEdge], node: Op, edge: Option[OpEdge]) = new Stream[T](graph, node, edge) + def apply[T](graph: Graph[Op, OpEdge], node: Op, edge: Option[OpEdge]): Stream[T] = { + new Stream[T](graph, node, edge) + } def getTupleKey[K, V](tuple: Tuple2[K, V]): K = tuple._1 def sumByValue[K, V](numeric: Numeric[V]): (Tuple2[K, V], Tuple2[K, V]) => Tuple2[K, V] = (tuple1, tuple2) => Tuple2(tuple1._1, numeric.plus(tuple1._2, tuple2._2)) - implicit def streamToKVStream[K, V](stream: Stream[Tuple2[K, V]]): KVStream[K, V] = new KVStream(stream) + implicit def streamToKVStream[K, V](stream: Stream[Tuple2[K, V]]): KVStream[K, V] = { + new KVStream(stream) + } implicit class Sink[T](stream: Stream[T]) extends java.io.Serializable { - def sink[T](dataSink: DataSink, parallism: Int, conf: UserConfig, description: String): Stream[T] = { - implicit val sink = DataSinkOp[T](dataSink, parallism, conf, Some(description).getOrElse("traversable")) + def sink[T](dataSink: DataSink, parallism: Int, conf: UserConfig, description: String) + : Stream[T] = { + implicit val sink = DataSinkOp[T](dataSink, parallism, conf, + Some(description).getOrElse("traversable")) stream.graph.addVertex(sink) stream.graph.addEdge(stream.thisNode, Shuffle, sink) new Stream[T](stream.graph, sink) } - def sink[T](sink: Class[_ <: Task], parallism: Int, conf: UserConfig = UserConfig.empty, description: String = null): Stream[T] = { + def sink[T]( + sink: Class[_ <: Task], parallism: Int, conf: UserConfig = UserConfig.empty, + description: String = null): Stream[T] = { val sinkOp = ProcessorOp(sink, parallism, conf, Option(description).getOrElse("source")) stream.graph.addVertex(sinkOp) stream.graph.addEdge(stream.thisNode, Shuffle, sinkOp) @@ -198,8 +218,7 @@ class LoggerSink[T] extends DataSink { private var context: TaskContext = null - - override def open(context: TaskContext) = { + override def open(context: TaskContext): Unit = { this.logger = context.logger } @@ -207,6 +226,5 @@ class LoggerSink[T] extends DataSink { logger.info("logging message " + message.msg) } - override def close() = Unit -} - + override def close(): Unit = Unit +} \ No newline at end of file diff --git a/streaming/src/main/scala/io/gearpump/streaming/dsl/StreamApp.scala b/streaming/src/main/scala/io/gearpump/streaming/dsl/StreamApp.scala index cfcfed204..525000d0b 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/dsl/StreamApp.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/dsl/StreamApp.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,36 +18,40 @@ package io.gearpump.streaming.dsl -import akka.actor.{ActorRef, ActorSystem, Cancellable, Props} +import scala.language.implicitConversions + +import akka.actor.ActorSystem + +import io.gearpump.cluster.UserConfig +import io.gearpump.cluster.client.ClientContext import io.gearpump.streaming.StreamApplication -import io.gearpump.streaming.dsl.op.{Shuffle, ProcessorOp, DataSourceOp, OpEdge, Op} +import io.gearpump.streaming.dsl.op.{DataSourceOp, Op, OpEdge, ProcessorOp} import io.gearpump.streaming.dsl.plan.Planner import io.gearpump.streaming.source.DataSource import io.gearpump.streaming.task.{Task, TaskContext} -import io.gearpump.cluster.UserConfig -import io.gearpump.cluster.client.ClientContext import io.gearpump.util.Graph import io.gearpump.{Message, TimeStamp} /** * Example: + * {{{ + * val data = "This is a good start, bingo!! bingo!!" + * app.fromCollection(data.lines.toList). + * // word => (word, count) + * flatMap(line => line.split("[\\s]+")).map((_, 1)). + * // (word, count1), (word, count2) => (word, count1 + count2) + * groupBy(kv => kv._1).reduce(sum(_, _)) * - * - val data = "This is a good start, bingo!! bingo!!" - app.fromCollection(data.lines.toList). - // word => (word, count) - flatMap(line => line.split("[\\s]+")).map((_, 1)). - // (word, count1), (word, count2) => (word, count1 + count2) - groupBy(kv => kv._1).reduce(sum(_, _)) - - val appId = context.submit(app) - context.close() + * val appId = context.submit(app) + * context.close() + * }}} * * @param name name of app */ -class StreamApp(val name: String, system: ActorSystem, userConfig: UserConfig, val graph: Graph[Op, OpEdge]) { +class StreamApp( + val name: String, system: ActorSystem, userConfig: UserConfig, val graph: Graph[Op, OpEdge]) { - def this(name: String ,system: ActorSystem, userConfig: UserConfig) = { + def this(name: String, system: ActorSystem, userConfig: UserConfig) = { this(name, system, userConfig, Graph.empty[Op, OpEdge]) } @@ -60,7 +64,10 @@ class StreamApp(val name: String, system: ActorSystem, userConfig: UserConfig, v } object StreamApp { - def apply(name: String, context: ClientContext, userConfig: UserConfig = UserConfig.empty) = new StreamApp(name, context.system, userConfig) + def apply(name: String, context: ClientContext, userConfig: UserConfig = UserConfig.empty) + : StreamApp = { + new StreamApp(name, context.system, userConfig) + } implicit def streamAppToApplication(streamApp: StreamApp): StreamApplication = { streamApp.plan @@ -80,7 +87,8 @@ object StreamApp { source(dataSource, parallism, conf, description = null) } - def source[T](dataSource: DataSource, parallism: Int, conf: UserConfig, description: String): Stream[T] = { + def source[T](dataSource: DataSource, parallism: Int, conf: UserConfig, description: String) + : Stream[T] = { implicit val sourceOp = DataSourceOp(dataSource, parallism, conf, description) app.graph.addVertex(sourceOp) new Stream[T](app.graph, sourceOp) @@ -89,7 +97,8 @@ object StreamApp { this.source(new CollectionDataSource[T](seq), parallism, UserConfig.empty, description) } - def source[T](source: Class[_ <: Task], parallism: Int, conf: UserConfig, description: String): Stream[T] = { + def source[T](source: Class[_ <: Task], parallism: Int, conf: UserConfig, description: String) + : Stream[T] = { val sourceOp = ProcessorOp(source, parallism, conf, Option(description).getOrElse("source")) app.graph.addVertex(sourceOp) new Stream[T](app.graph, sourceOp) @@ -97,6 +106,7 @@ object StreamApp { } } +/** A test message source which generated message sequence repeatedly. */ class CollectionDataSource[T](seq: Seq[T]) extends DataSource { val list = seq.toList var index = 0 diff --git a/streaming/src/main/scala/io/gearpump/streaming/dsl/javaapi/JavaStream.scala b/streaming/src/main/scala/io/gearpump/streaming/dsl/javaapi/JavaStream.scala index 03bfa81da..549cc6ee0 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/dsl/javaapi/JavaStream.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/dsl/javaapi/JavaStream.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,47 +18,60 @@ package io.gearpump.streaming.dsl.javaapi +import scala.collection.JavaConverters._ + import io.gearpump.cluster.UserConfig import io.gearpump.streaming.dsl.Stream import io.gearpump.streaming.javaapi.dsl.functions._ import io.gearpump.streaming.task.Task -import scala.collection.JavaConverters._ - /** * Java DSL */ class JavaStream[T](val stream: Stream[T]) { + /** FlatMap on stream */ def flatMap[R](fn: FlatMapFunction[T, R], description: String): JavaStream[R] = { - new JavaStream[R](stream.flatMap({t: T => fn(t).asScala}, description)) + new JavaStream[R](stream.flatMap({ t: T => fn(t).asScala }, description)) } + /** Map on stream */ def map[R](fn: MapFunction[T, R], description: String): JavaStream[R] = { - new JavaStream[R](stream.map({t: T => fn(t)}, description)) + new JavaStream[R](stream.map({ t: T => fn(t) }, description)) } + /** Only keep the messages that FilterFunction returns true. */ def filter(fn: FilterFunction[T], description: String): JavaStream[T] = { - new JavaStream[T](stream.filter({t: T => fn(t)}, description)) + new JavaStream[T](stream.filter({ t: T => fn(t) }, description)) } + /** Does aggregation on the stream */ def reduce(fn: ReduceFunction[T], description: String): JavaStream[T] = { - new JavaStream[T](stream.reduce({(t1: T, t2: T) => fn(t1, t2)}, description)) + new JavaStream[T](stream.reduce({ (t1: T, t2: T) => fn(t1, t2) }, description)) } def log(): Unit = { stream.log() } + /** Merges streams of same type together */ def merge(other: JavaStream[T], description: String): JavaStream[T] = { new JavaStream[T](stream.merge(other.stream, description)) } - def groupBy[Group](fn: GroupByFunction[T, Group], parallelism: Int, description: String): JavaStream[T] = { + /** + * Group by a stream and turns it to a list of sub-streams. Operations chained after + * groupBy applies to sub-streams. + */ + def groupBy[Group](fn: GroupByFunction[T, Group], parallelism: Int, description: String) + : JavaStream[T] = { new JavaStream[T](stream.groupBy({t: T => fn(t)}, parallelism, description)) } - def process[R](processor: Class[_ <: Task], parallelism: Int, conf: UserConfig, description: String): JavaStream[R] = { + /** Add a low level Processor to process messages */ + def process[R]( + processor: Class[_ <: Task], parallelism: Int, conf: UserConfig, description: String) + : JavaStream[R] = { new JavaStream[R](stream.process(processor, parallelism, conf, description)) } } diff --git a/streaming/src/main/scala/io/gearpump/streaming/dsl/javaapi/JavaStreamApp.scala b/streaming/src/main/scala/io/gearpump/streaming/dsl/javaapi/JavaStreamApp.scala index 0ad03cdca..e39e0547f 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/dsl/javaapi/JavaStreamApp.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/dsl/javaapi/JavaStreamApp.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,31 +19,29 @@ package io.gearpump.streaming.dsl.javaapi import java.util.Collection +import scala.collection.JavaConverters._ import io.gearpump.cluster.UserConfig import io.gearpump.cluster.client.ClientContext import io.gearpump.streaming.dsl.{CollectionDataSource, StreamApp} import io.gearpump.streaming.source.DataSource -import scala.collection.JavaConverters._ - class JavaStreamApp(name: String, context: ClientContext, userConfig: UserConfig) { private val streamApp = StreamApp(name, context, userConfig) def source[T](collection: Collection[T], parallelism: Int, - conf: UserConfig, description: String): JavaStream[T] = { + conf: UserConfig, description: String): JavaStream[T] = { val dataSource = new CollectionDataSource(collection.asScala.toSeq) source(dataSource, parallelism, conf, description) } def source[T](dataSource: DataSource, parallelism: Int, - conf: UserConfig, description: String): JavaStream[T] = { + conf: UserConfig, description: String): JavaStream[T] = { new JavaStream[T](streamApp.source(dataSource, parallelism, conf, description)) } def run(): Unit = { context.submit(streamApp) } - } diff --git a/streaming/src/main/scala/io/gearpump/streaming/dsl/op/OP.scala b/streaming/src/main/scala/io/gearpump/streaming/dsl/op/OP.scala index e0b89cea8..f0a86fabc 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/dsl/op/OP.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/dsl/op/OP.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -34,27 +34,39 @@ sealed trait Op { /** * When translated to running DAG, SlaveOP can be attach to MasterOP or other SlaveOP * "Attach" means running in same Actor. - * */ trait SlaveOp[T] extends Op -case class FlatMapOp[T, R](fun: (T) => TraversableOnce[R], description: String, conf: UserConfig = UserConfig.empty) extends SlaveOp[T] +case class FlatMapOp[T, R]( + fun: (T) => TraversableOnce[R], description: String, conf: UserConfig = UserConfig.empty) + extends SlaveOp[T] -case class ReduceOp[T](fun: (T, T) =>T, description: String, conf: UserConfig = UserConfig.empty) extends SlaveOp[T] +case class ReduceOp[T](fun: (T, T) => T, description: String, conf: UserConfig = UserConfig.empty) + extends SlaveOp[T] trait MasterOp extends Op trait ParameterizedOp[T] extends MasterOp -case class MergeOp(description: String, override val conf: UserConfig = UserConfig.empty) extends MasterOp +case class MergeOp(description: String, override val conf: UserConfig = UserConfig.empty) + extends MasterOp -case class GroupByOp[T, R](fun: T => R, parallelism: Int, description: String, override val conf: UserConfig = UserConfig.empty) extends ParameterizedOp[T] +case class GroupByOp[T, R]( + fun: T => R, parallelism: Int, description: String, + override val conf: UserConfig = UserConfig.empty) + extends ParameterizedOp[T] -case class ProcessorOp[T <: Task](processor: Class[T], parallelism: Int, conf: UserConfig, description: String) extends ParameterizedOp[T] +case class ProcessorOp[T <: Task]( + processor: Class[T], parallelism: Int, conf: UserConfig, description: String) + extends ParameterizedOp[T] -case class DataSourceOp[T](dataSource: DataSource, parallelism: Int, conf: UserConfig, description: String) extends ParameterizedOp[T] +case class DataSourceOp[T]( + dataSource: DataSource, parallelism: Int, conf: UserConfig, description: String) + extends ParameterizedOp[T] -case class DataSinkOp[T](dataSink: DataSink, parallelism: Int, conf: UserConfig, description: String) extends ParameterizedOp[T] +case class DataSinkOp[T]( + dataSink: DataSink, parallelism: Int, conf: UserConfig, description: String) + extends ParameterizedOp[T] /** * Contains operators which can be chained to single one. @@ -70,8 +82,8 @@ case class OpChain(ops: List[Op]) extends Op { def description: String = null override def conf: UserConfig = { - // the head's conf has priority - ops.reverse.foldLeft(UserConfig.empty){(conf, op) => + // The head's conf has priority + ops.reverse.foldLeft(UserConfig.empty) { (conf, op) => conf.withConfig(op.conf) } } @@ -84,7 +96,6 @@ trait OpEdge * * For example, map, flatmap operation doesn't require network shuffle, we can use Direct * to represent the relation with upstream operators. - * */ case object Direct extends OpEdge @@ -93,7 +104,6 @@ case object Direct extends OpEdge * * For example, map, flatmap operation doesn't require network shuffle, we can use Direct * to represent the relation with upstream operators. - * */ case object Shuffle extends OpEdge diff --git a/streaming/src/main/scala/io/gearpump/streaming/dsl/partitioner/GroupbyPartitioner.scala b/streaming/src/main/scala/io/gearpump/streaming/dsl/partitioner/GroupbyPartitioner.scala index 7dad9cc38..b842c7ba1 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/dsl/partitioner/GroupbyPartitioner.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/dsl/partitioner/GroupbyPartitioner.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,13 +20,12 @@ package io.gearpump.streaming.dsl.partitioner import io.gearpump.Message import io.gearpump.partitioner.UnicastPartitioner -/** Partition messages by applying group by function first. - * - * @param groupBy - * First apply message with groupBy function, then pick the hashCode of the output to do the partitioning. - * You must define hashCode() for output type of groupBy function. + +/** + * Partition messages by applying group by function first. * * For example: + * {{{ * case class People(name: String, gender: String) * * object Test{ @@ -34,9 +33,13 @@ import io.gearpump.partitioner.UnicastPartitioner * val groupBy: (People => String) = people => people.gender * val partitioner = GroupByPartitioner(groupBy) * } + * }}} + * + * @param groupBy First apply message with groupBy function, then pick the hashCode of the output + * to do the partitioning. You must define hashCode() for output type of groupBy function. */ class GroupByPartitioner[T, GROUP](groupBy: T => GROUP = null) extends UnicastPartitioner { - override def getPartition(msg : Message, partitionNum : Int, currentPartitionId: Int) : Int = { + override def getPartition(msg: Message, partitionNum: Int, currentPartitionId: Int): Int = { val hashCode = groupBy(msg.msg.asInstanceOf[T]).hashCode() (hashCode & Integer.MAX_VALUE) % partitionNum } diff --git a/streaming/src/main/scala/io/gearpump/streaming/dsl/plan/OpTranslator.scala b/streaming/src/main/scala/io/gearpump/streaming/dsl/plan/OpTranslator.scala index 400c9b1b2..f91612436 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/dsl/plan/OpTranslator.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/dsl/plan/OpTranslator.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,24 +18,25 @@ package io.gearpump.streaming.dsl.plan +import scala.collection.TraversableOnce + import akka.actor.ActorSystem -import io.gearpump.streaming.sink.DataSink -import io.gearpump.streaming.source.DataSource -import io.gearpump.streaming.{Processor, Constants} -import io.gearpump.streaming.dsl.op._ -import io.gearpump.streaming.task.{StartTime, TaskContext, Task} +import org.slf4j.Logger + import io.gearpump._ import io.gearpump.cluster.UserConfig -import Constants._ -import Processor.DefaultProcessor -import OpTranslator._ +import io.gearpump.streaming.Constants._ +import io.gearpump.streaming.Processor +import io.gearpump.streaming.Processor.DefaultProcessor +import io.gearpump.streaming.dsl.op._ +import io.gearpump.streaming.dsl.plan.OpTranslator._ +import io.gearpump.streaming.sink.DataSink +import io.gearpump.streaming.source.DataSource +import io.gearpump.streaming.task.{StartTime, Task, TaskContext} import io.gearpump.util.LogUtil -import org.slf4j.Logger - -import scala.collection.TraversableOnce /** - * Translate a OP to a TaskDescription + * Translates a OP to a TaskDescription */ class OpTranslator extends java.io.Serializable { val LOG: Logger = LogUtil.getLogger(getClass) @@ -55,7 +56,7 @@ class OpTranslator extends java.io.Serializable { Processor[SourceTask[Object, Object]](parallism, description = description + "." + func.description, userConfig.withValue(GEARPUMP_STREAMING_SOURCE, dataSource)) - case groupby@ GroupByOp(_, parallism, description, _) => + case groupby@GroupByOp(_, parallism, description, _) => Processor[GroupByTask[Object, Object, Object]](parallism, description = description + "." + func.description, userConfig.withValue(GEARPUMP_STREAMING_GROUPBY_FUNCTION, groupby)) @@ -79,6 +80,8 @@ class OpTranslator extends java.io.Serializable { Processor[TransformTask[Object, Object]](1, description = func.description, taskConf = userConfig) + case chain: OpChain => + throw new RuntimeException("Not supposed to be called!") } } @@ -87,8 +90,12 @@ class OpTranslator extends java.io.Serializable { val totalFunction = ops.foldLeft(func) { (fun, op) => val opFunction = op match { - case flatmap: FlatMapOp[Object, Object] => new FlatMapFunction(flatmap.fun, flatmap.description) - case reduce: ReduceOp[Object] => new ReduceFunction(reduce.fun, reduce.description) + case flatmap: FlatMapOp[Object @unchecked, Object @unchecked] => + new FlatMapFunction(flatmap.fun, flatmap.description) + case reduce: ReduceOp[Object @unchecked] => + new ReduceFunction(reduce.fun, reduce.description) + case _ => + throw new RuntimeException("Not supposed to be called!") } fun.andThen(opFunction.asInstanceOf[SingleInputFunction[Object, Object]]) } @@ -107,18 +114,22 @@ object OpTranslator { def description: String } - class DummyInputFunction[T] extends SingleInputFunction[T, T]{ - override def andThen[OUTER](other: SingleInputFunction[T, OUTER]): SingleInputFunction[T, OUTER] = { + class DummyInputFunction[T] extends SingleInputFunction[T, T] { + override def andThen[OUTER](other: SingleInputFunction[T, OUTER]) + : SingleInputFunction[T, OUTER] = { other } - //should never be called - override def process(value: T) = None + // Should never be called + override def process(value: T): TraversableOnce[T] = None override def description: String = "" } - class AndThen[IN, MIDDLE, OUT](first: SingleInputFunction[IN, MIDDLE], second: SingleInputFunction[MIDDLE, OUT]) extends SingleInputFunction[IN, OUT] { + class AndThen[IN, MIDDLE, OUT]( + first: SingleInputFunction[IN, MIDDLE], second: SingleInputFunction[MIDDLE, OUT]) + extends SingleInputFunction[IN, OUT] { + override def process(value: IN): TraversableOnce[OUT] = { first.process(value).flatMap(second.process(_)) } @@ -130,7 +141,9 @@ object OpTranslator { } } - class FlatMapFunction[IN, OUT](fun: IN => TraversableOnce[OUT], descriptionMessage: String) extends SingleInputFunction[IN, OUT] { + class FlatMapFunction[IN, OUT](fun: IN => TraversableOnce[OUT], descriptionMessage: String) + extends SingleInputFunction[IN, OUT] { + override def process(value: IN): TraversableOnce[OUT] = { fun(value) } @@ -140,7 +153,9 @@ object OpTranslator { } } - class ReduceFunction[T](fun: (T, T)=>T, descriptionMessage: String) extends SingleInputFunction[T, T] { + class ReduceFunction[T](fun: (T, T) => T, descriptionMessage: String) + extends SingleInputFunction[T, T] { + private var state: Any = null override def process(value: T): TraversableOnce[T] = { @@ -155,10 +170,13 @@ object OpTranslator { override def description: String = descriptionMessage } - class GroupByTask[IN, GROUP, OUT](groupBy: IN => GROUP, taskContext: TaskContext, userConf: UserConfig) extends Task(taskContext, userConf) { + class GroupByTask[IN, GROUP, OUT]( + groupBy: IN => GROUP, taskContext: TaskContext, userConf: UserConfig) + extends Task(taskContext, userConf) { def this(taskContext: TaskContext, userConf: UserConfig) = { - this(userConf.getValue[GroupByOp[IN, GROUP]](GEARPUMP_STREAMING_GROUPBY_FUNCTION )(taskContext.system).get.fun, + this(userConf.getValue[GroupByOp[IN, GROUP]]( + GEARPUMP_STREAMING_GROUPBY_FUNCTION )(taskContext.system).get.fun, taskContext, userConf) } @@ -172,24 +190,29 @@ object OpTranslator { val group = groupBy(msg.msg.asInstanceOf[IN]) if (!groups.contains(group)) { - val operator = userConf.getValue[SingleInputFunction[IN, OUT]](GEARPUMP_STREAMING_OPERATOR).get + val operator = + userConf.getValue[SingleInputFunction[IN, OUT]](GEARPUMP_STREAMING_OPERATOR).get groups += group -> operator } val operator = groups(group) - operator.process(msg.msg.asInstanceOf[IN]).foreach{msg => + operator.process(msg.msg.asInstanceOf[IN]).foreach { msg => taskContext.output(new Message(msg.asInstanceOf[AnyRef], time)) } } } - class SourceTask[T, OUT](source: DataSource, operator: Option[SingleInputFunction[T, OUT]], taskContext: TaskContext, userConf: UserConfig) extends Task(taskContext, userConf) { + class SourceTask[T, OUT]( + source: DataSource, operator: Option[SingleInputFunction[T, OUT]], taskContext: TaskContext, + userConf: UserConfig) + extends Task(taskContext, userConf) { def this(taskContext: TaskContext, userConf: UserConfig) = { this( userConf.getValue[DataSource](GEARPUMP_STREAMING_SOURCE)(taskContext.system).get, - userConf.getValue[SingleInputFunction[T, OUT]](GEARPUMP_STREAMING_OPERATOR)(taskContext.system), + userConf.getValue[SingleInputFunction[T, OUT]](GEARPUMP_STREAMING_OPERATOR)( + taskContext.system), taskContext, userConf) } @@ -200,7 +223,7 @@ object OpTranslator { override def onNext(msg: Message): Unit = { val time = System.currentTimeMillis() - //Todo: determine the batch size + // TODO: determine the batch size source.read(1).foreach(msg => { operator match { case Some(operator) => @@ -219,15 +242,18 @@ object OpTranslator { self ! Message("next", System.currentTimeMillis()) } - override def onStop() = { + override def onStop(): Unit = { source.close() } } - class TransformTask[IN, OUT](operator: Option[SingleInputFunction[IN, OUT]], taskContext: TaskContext, userConf: UserConfig) extends Task(taskContext, userConf) { + class TransformTask[IN, OUT]( + operator: Option[SingleInputFunction[IN, OUT]], taskContext: TaskContext, + userConf: UserConfig) extends Task(taskContext, userConf) { def this(taskContext: TaskContext, userConf: UserConfig) = { - this(userConf.getValue[SingleInputFunction[IN, OUT]](GEARPUMP_STREAMING_OPERATOR)(taskContext.system), taskContext, userConf) + this(userConf.getValue[SingleInputFunction[IN, OUT]]( + GEARPUMP_STREAMING_OPERATOR)(taskContext.system), taskContext, userConf) } override def onStart(startTime: StartTime): Unit = { @@ -238,7 +264,7 @@ object OpTranslator { operator match { case Some(operator) => - operator.process(msg.msg.asInstanceOf[IN]).foreach{ msg => + operator.process(msg.msg.asInstanceOf[IN]).foreach { msg => taskContext.output(new Message(msg.asInstanceOf[AnyRef], time)) } case None => @@ -247,9 +273,12 @@ object OpTranslator { } } - class SinkTask[T](dataSink: DataSink, taskContext: TaskContext, userConf: UserConfig) extends Task(taskContext, userConf) { + class SinkTask[T](dataSink: DataSink, taskContext: TaskContext, userConf: UserConfig) + extends Task(taskContext, userConf) { + def this(taskContext: TaskContext, userConf: UserConfig) = { - this(userConf.getValue[DataSink](GEARPUMP_STREAMING_SINK)(taskContext.system).get, taskContext, userConf) + this(userConf.getValue[DataSink](GEARPUMP_STREAMING_SINK)(taskContext.system).get, + taskContext, userConf) } override def onStart(startTime: StartTime): Unit = { @@ -260,7 +289,7 @@ object OpTranslator { dataSink.write(msg) } - override def onStop() = { + override def onStop(): Unit = { dataSink.close() } } diff --git a/streaming/src/main/scala/io/gearpump/streaming/dsl/plan/Planner.scala b/streaming/src/main/scala/io/gearpump/streaming/dsl/plan/Planner.scala index ee0818d4e..aafd8d39c 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/dsl/plan/Planner.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/dsl/plan/Planner.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,35 +19,38 @@ package io.gearpump.streaming.dsl.plan import akka.actor.ActorSystem + +import io.gearpump.partitioner.{CoLocationPartitioner, HashPartitioner, Partitioner} import io.gearpump.streaming.Processor import io.gearpump.streaming.dsl.op._ import io.gearpump.streaming.dsl.partitioner.GroupByPartitioner import io.gearpump.streaming.task.Task -import io.gearpump.partitioner.{CoLocationPartitioner, HashPartitioner, Partitioner} import io.gearpump.util.Graph class Planner { /* - * Convert Dag[Op] to Dag[TaskDescription] so that we can run it easily. + * Converts Dag of Op to Dag of TaskDescription. TaskDescription is part of the low + * level Graph API. */ - def plan(dag: Graph[Op, OpEdge])(implicit system: ActorSystem): Graph[Processor[_ <: Task], _ <: Partitioner] = { + def plan(dag: Graph[Op, OpEdge])(implicit system: ActorSystem) + : Graph[Processor[_ <: Task], _ <: Partitioner] = { val opTranslator = new OpTranslator() val newDag = optimize(dag) - newDag.mapEdge {(node1, edge, node2) => + newDag.mapEdge { (node1, edge, node2) => edge match { case Shuffle => node2.head match { - case groupBy: GroupByOp[Any, Any] => + case groupBy: GroupByOp[Any @unchecked, Any @unchecked] => new GroupByPartitioner(groupBy.fun) case _ => new HashPartitioner } case Direct => new CoLocationPartitioner } - }.mapVertex {opChain => + }.mapVertex { opChain => opTranslator.translate(opChain) } } @@ -65,11 +68,12 @@ class Planner { newGraph } - private def merge(dag: Graph[OpChain, OpEdge], node1: OpChain, node2: OpChain): Graph[OpChain, OpEdge] = { + private def merge(dag: Graph[OpChain, OpEdge], node1: OpChain, node2: OpChain) + : Graph[OpChain, OpEdge] = { if (dag.outDegreeOf(node1) == 1 && dag.inDegreeOf(node2) == 1 && - // for processor node, we don't allow it to merge with downstream operators - !node1.head.isInstanceOf[ProcessorOp[_<:Task]]) { + // For processor node, we don't allow it to merge with downstream operators + !node1.head.isInstanceOf[ProcessorOp[_ <: Task]]) { val (_, edge, _) = dag.outgoingEdgesOf(node1)(0) if (edge == Direct) { val opList = OpChain(node1.ops ++ node2.ops) @@ -82,7 +86,7 @@ class Planner { dag.addEdge(opList, outgoingEdge._2, outgoingEdge._3) } - //remove the old vertex + // Remove the old vertex dag.removeVertex(node1) dag.removeVertex(node2) } diff --git a/streaming/src/main/scala/io/gearpump/streaming/executor/Executor.scala b/streaming/src/main/scala/io/gearpump/streaming/executor/Executor.scala index cbce65b9e..48007d5cb 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/executor/Executor.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/executor/Executor.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,17 +19,21 @@ package io.gearpump.streaming.executor import java.lang.management.ManagementFactory +import scala.concurrent.duration._ import akka.actor.SupervisorStrategy.Resume import akka.actor._ import com.typesafe.config.Config -import io.gearpump.WorkerId +import org.apache.commons.lang.exception.ExceptionUtils +import org.slf4j.Logger + +import io.gearpump.cluster.worker.WorkerId import io.gearpump.cluster.{ClusterConfig, ExecutorContext, UserConfig} import io.gearpump.metrics.Metrics.ReportMetrics import io.gearpump.metrics.{JvmMetricsSet, Metrics, MetricsReporterService} import io.gearpump.serializer.SerializationFramework import io.gearpump.streaming.AppMasterToExecutor.{MsgLostException, TasksChanged, TasksLaunched, _} -import io.gearpump.streaming.ExecutorToAppMaster.{UnRegisterTask, MessageLoss, RegisterExecutor, RegisterTask} +import io.gearpump.streaming.ExecutorToAppMaster.{MessageLoss, RegisterExecutor, RegisterTask, UnRegisterTask} import io.gearpump.streaming.ProcessorId import io.gearpump.streaming.executor.Executor._ import io.gearpump.streaming.executor.TaskLauncher.TaskArgument @@ -37,11 +41,6 @@ import io.gearpump.streaming.task.{Subscriber, TaskId} import io.gearpump.transport.{Express, HostPort} import io.gearpump.util.Constants._ import io.gearpump.util.{ActorUtil, Constants, LogUtil, TimeOutScheduler} -import org.apache.commons.lang.exception.ExceptionUtils -import org.slf4j.Logger - -import scala.concurrent.duration._ -import scala.language.postfixOps /** * Executor is child of AppMaster. @@ -64,18 +63,20 @@ class Executor(executorContext: ExecutorContext, userConf : UserConfig, launcher private val LOG: Logger = LogUtil.getLogger(getClass, executor = executorId, app = appId) - implicit val timeOut = FUTURE_TIMEOUT + private implicit val timeOut = FUTURE_TIMEOUT private val address = ActorUtil.getFullPath(context.system, self.path) private val systemConfig = context.system.settings.config private val serializerPool = getSerializerPool() private val taskDispatcher = systemConfig.getString(Constants.GEARPUMP_TASK_DISPATCHER) private var state = State.ACTIVE - private var transitionStart = 0L // state transition start, in unix time - private var transitionEnd = 0L // state transition end, in unix time + private var transitionStart = 0L + // States transition start, in unix time + private var transitionEnd = 0L + // States transition end, in unix time private val transitWarningThreshold = 5000 // ms, - // start health check Ticks + // Starts health check Ticks self ! HealthCheck LOG.info(s"Executor $executorId has been started, start to register itself...") @@ -92,31 +93,34 @@ class Executor(executorContext: ExecutorContext, userConf : UserConfig, launcher val metricsEnabled = systemConfig.getBoolean(GEARPUMP_METRIC_ENABLED) if (metricsEnabled) { - // register jvm metrics + // Registers jvm metrics Metrics(context.system).register(new JvmMetricsSet(s"app$appId.executor$executorId")) - val metricsReportService = context.actorOf(Props(new MetricsReporterService(Metrics(context.system)))) + val metricsReportService = context.actorOf(Props(new MetricsReporterService( + Metrics(context.system)))) appMaster.tell(ReportMetrics, metricsReportService) } private val NOT_INITIALIZED = -1 - def receive : Receive = applicationReady(dagVersion = NOT_INITIALIZED) + def receive: Receive = applicationReady(dagVersion = NOT_INITIALIZED) private def getTaskId(actorRef: ActorRef): Option[TaskId] = { tasks.find(_._2 == actorRef).map(_._1) } override val supervisorStrategy = - OneForOneStrategy(maxNrOfRetries = 10, withinTimeRange = 1 minute) { + OneForOneStrategy(maxNrOfRetries = 10, withinTimeRange = 1.minute) { case _: MsgLostException => val taskId = getTaskId(sender) - val cause = s"We got MessageLossException from task ${getTaskId(sender)}, replaying application..." + val cause = s"We got MessageLossException from task ${getTaskId(sender)}, " + + s"replaying application..." LOG.error(cause) - taskId.foreach(appMaster ! MessageLoss(executorId, _, cause)) + taskId.foreach(appMaster ! MessageLoss(executorId, _, cause)) Resume case ex: Throwable => val taskId = getTaskId(sender) - val errorMsg = s"We got ${ex.getClass.getName} from $taskId, we will treat it as MessageLoss, so that the system will replay all lost message" + val errorMsg = s"We got ${ex.getClass.getName} from $taskId, we will treat it as" + + s" MessageLoss, so that the system will replay all lost message" LOG.error(errorMsg, ex) val detailErrorMsg = errorMsg + "\n" + ExceptionUtils.getStackTrace(ex) taskId.foreach(appMaster ! MessageLoss(executorId, _, detailErrorMsg)) @@ -129,22 +133,27 @@ class Executor(executorContext: ExecutorContext, userConf : UserConfig, launcher private def assertVersion(expectVersion: Int, version: Int, clue: Any): Unit = { if (expectVersion != version) { - val errorMessage = s"Version mismatch: we expect dag version $expectVersion, but get $version; clue: $clue" + val errorMessage = s"Version mismatch: we expect dag version $expectVersion, " + + s"but get $version; clue: $clue" LOG.error(errorMessage) throw new DagVersionMismatchException(errorMessage) } } - def dynamicDagPhase1(dagVersion: Int, launched: List[TaskId], changed: List[ChangeTask], registered: List[TaskId]): Receive = { + def dynamicDagPhase1( + dagVersion: Int, launched: List[TaskId], changed: List[ChangeTask], registered: List[TaskId]) + : Receive = { state = State.DYNAMIC_DAG_PHASE1 box({ - case launch@LaunchTasks(taskIds, version, processorDescription, subscribers: List[Subscriber]) => { + case launch@LaunchTasks(taskIds, version, processorDescription, + subscribers: List[Subscriber]) => { assertVersion(dagVersion, version, clue = launch) LOG.info(s"Launching Task $taskIds for app: $appId") val taskArgument = TaskArgument(version, processorDescription, subscribers) taskIds.foreach(taskArgumentStore.add(_, taskArgument)) - val newAdded = launcher.launch(taskIds, taskArgument, context, serializerPool, taskDispatcher) + val newAdded = launcher.launch(taskIds, taskArgument, context, serializerPool, + taskDispatcher) newAdded.foreach { newAddedTask => context.watch(newAddedTask._2) } @@ -160,12 +169,14 @@ class Executor(executorContext: ExecutorContext, userConf : UserConfig, launcher val newChangedTasks = taskIds.map { taskId => for (taskArgument <- taskArgumentStore.get(dagVersion, taskId)) { val processorDescription = taskArgument.processorDescription.copy(life = life) - taskArgumentStore.add(taskId, TaskArgument(dagVersion, processorDescription, subscribers)) + taskArgumentStore.add(taskId, TaskArgument(dagVersion, processorDescription, + subscribers)) } ChangeTask(taskId, dagVersion, life, subscribers) } sender ! TasksChanged(taskIds) - context.become(dynamicDagPhase1(dagVersion, launched, changed ++ newChangedTasks, registered)) + context.become(dynamicDagPhase1(dagVersion, launched, changed ++ newChangedTasks, + registered)) case locations@TaskLocationsReady(taskLocations, version) => LOG.info(s"TaskLocations Ready...") @@ -174,7 +185,9 @@ class Executor(executorContext: ExecutorContext, userConf : UserConfig, launcher // Check whether all tasks has been registered. if ((launched.toSet -- registered.toSet).isEmpty) { // Confirm all tasks has been registered. - val result = taskLocations.locations.filter(location => !location._1.equals(express.localHost)).flatMap { kv => + val result = taskLocations.locations.filter { + location => !location._1.equals(express.localHost) + }.flatMap { kv => val (host, taskIdList) = kv taskIdList.map(taskId => (TaskId.toLong(taskId), host)) } @@ -189,14 +202,17 @@ class Executor(executorContext: ExecutorContext, userConf : UserConfig, launcher } context.become(dynamicDagPhase2(dagVersion, launched, changed)) } else { - LOG.error("Inconsistency between AppMaser and Executor! AppMaster thinks DynamicDag transition is ready, " + - "while Executor have not get all tasks registered, that task will not be functional...") - //reject TaskLocations... + LOG.error("Inconsistency between AppMaser and Executor! AppMaster thinks DynamicDag " + + "transition is ready, while Executor have not get all tasks registered, " + + "that task will not be functional...") + + // Reject TaskLocations... val missedTasks = (launched.toSet -- registered.toSet).toList - val errorMsg = "We have not received TaskRegistered for following tasks: " + missedTasks.mkString(", ") + val errorMsg = "We have not received TaskRegistered for following tasks: " + + missedTasks.mkString(", ") LOG.error(errorMsg) sender ! TaskLocationsRejected(dagVersion, executorId, errorMsg, null) - // stay with current status... + // Stays with current status... } case confirm: TaskRegistered => @@ -205,10 +221,11 @@ class Executor(executorContext: ExecutorContext, userConf : UserConfig, launcher tasks += confirm.taskId -> actorRef actorRef forward confirm } - context.become(dynamicDagPhase1(dagVersion, launched, changed, registered :+ confirm.taskId)) + context.become(dynamicDagPhase1(dagVersion, launched, changed, + registered :+ confirm.taskId)) case rejected: TaskRejected => - // means this task shoud not exists... + // Means this task shoud not exists... tasks.get(rejected.taskId).foreach(_ ! PoisonPill) tasks -= rejected.taskId LOG.error(s"Task ${rejected.taskId} is rejected by AppMaster, shutting down it...") @@ -218,7 +235,8 @@ class Executor(executorContext: ExecutorContext, userConf : UserConfig, launcher }) } - def dynamicDagPhase2(dagVersion: Int, launched: List[TaskId], changed: List[ChangeTask]): Receive = { + def dynamicDagPhase2(dagVersion: Int, launched: List[TaskId], changed: List[ChangeTask]) + : Receive = { LOG.info("Transit to dynamic Dag Phase2") state = State.DYNAMIC_DAG_PHASE2 box { @@ -240,29 +258,36 @@ class Executor(executorContext: ExecutorContext, userConf : UserConfig, launcher transitionEnd = System.currentTimeMillis() if (dagVersion != NOT_INITIALIZED) { - LOG.info("Transit to state Application Ready. This transition takes " + (transitionEnd - transitionStart) + " milliseconds") + LOG.info("Transit to state Application Ready. This transition takes " + + (transitionEnd - transitionStart) + " milliseconds") } box { case start: StartDynamicDag => LOG.info("received StartDynamicDag") if (start.dagVersion > dagVersion) { transitionStart = System.currentTimeMillis() - LOG.info(s"received $start, Executor transit to dag version: ${start.dagVersion} from current version $dagVersion") - context.become(dynamicDagPhase1(start.dagVersion, List.empty[TaskId], List.empty[ChangeTask], List.empty[TaskId])) + LOG.info(s"received $start, Executor transit to dag version: ${start.dagVersion} from " + + s"current version $dagVersion") + context.become(dynamicDagPhase1(start.dagVersion, List.empty[TaskId], + List.empty[ChangeTask], List.empty[TaskId])) } case launch: LaunchTasks => if (launch.dagVersion > dagVersion) { transitionStart = System.currentTimeMillis() - LOG.info(s"received $launch, Executor transit to dag version: ${launch.dagVersion} from current version $dagVersion") - context.become(dynamicDagPhase1(launch.dagVersion, List.empty[TaskId], List.empty[ChangeTask], List.empty[TaskId])) + LOG.info(s"received $launch, Executor transit to dag " + + s"version: ${launch.dagVersion} from current version $dagVersion") + context.become(dynamicDagPhase1(launch.dagVersion, List.empty[TaskId], + List.empty[ChangeTask], List.empty[TaskId])) self forward launch } case change: ChangeTasks => if (change.dagVersion > dagVersion) { transitionStart = System.currentTimeMillis() - LOG.info(s"received $change, Executor transit to dag version: ${change.dagVersion} from current version $dagVersion") - context.become(dynamicDagPhase1(change.dagVersion, List.empty[TaskId], List.empty[ChangeTask], List.empty[TaskId])) + LOG.info(s"received $change, Executor transit to dag version: ${change.dagVersion} from" + + s" current version $dagVersion") + context.become(dynamicDagPhase1(change.dagVersion, List.empty[TaskId], + List.empty[ChangeTask], List.empty[TaskId])) self forward change } @@ -274,8 +299,8 @@ class Executor(executorContext: ExecutorContext, userConf : UserConfig, launcher } tasks -= taskId - case unRegister @ UnRegisterTask(taskId, _) => - // send UnRegisterTask to AppMaster + case unRegister@UnRegisterTask(taskId, _) => + // Sends UnRegisterTask to AppMaster appMaster ! unRegister } } @@ -289,14 +314,15 @@ class Executor(executorContext: ExecutorContext, userConf : UserConfig, launcher val newNeedRestart = needRestart :+ taskId val newRemain = remain - 1 if (newRemain == 0) { - val newRestarted = newNeedRestart.map{ taskId_ => + val newRestarted = newNeedRestart.map { taskId_ => val taskActor = launchTask(taskId_, taskArgumentStore.get(dagVersion, taskId_).get) context.watch(taskActor) taskId_ -> taskActor }.toMap tasks = newRestarted - context.become(dynamicDagPhase1(dagVersion, newNeedRestart, List.empty[ChangeTask], List.empty[TaskId])) + context.become(dynamicDagPhase1(dagVersion, newNeedRestart, List.empty[ChangeTask], + List.empty[TaskId])) } else { context.become(restartingTasks(dagVersion, newRemain, newNeedRestart)) } @@ -308,7 +334,8 @@ class Executor(executorContext: ExecutorContext, userConf : UserConfig, launcher val terminationWatch: Receive = { case Terminated(actor) => if (actor.compareTo(appMaster) == 0) { - LOG.info(s"AppMaster ${appMaster.path.toString} is terminated, shutting down current executor $appId, $executorId") + LOG.info(s"AppMaster ${appMaster.path.toString} is terminated, shutting down current " + + s"executor $appId, $executorId") context.stop(self) } else { self ! TaskStopped(actor) @@ -320,7 +347,8 @@ class Executor(executorContext: ExecutorContext, userConf : UserConfig, launcher LOG.info(s"Executor received restart tasks") val tasksToRestart = tasks.keys.count(taskArgumentStore.get(dagVersion, _).nonEmpty) express.remoteAddressMap.send(Map.empty[Long, HostPort]) - context.become(restartingTasks(dagVersion, remain = tasksToRestart, needRestart = List.empty[TaskId])) + context.become(restartingTasks(dagVersion, remain = tasksToRestart, + needRestart = List.empty[TaskId])) tasks.values.foreach { case task: ActorRef => task ! PoisonPill @@ -329,7 +357,7 @@ class Executor(executorContext: ExecutorContext, userConf : UserConfig, launcher def executorService: Receive = terminationWatch orElse onRestartTasks orElse { case taskChanged: TaskChanged => - //skip + // Skip case get: GetExecutorSummary => val logFile = LogUtil.applicationLogDir(systemConfig) val processorTasks = tasks.keySet.groupBy(_.processorId).mapValues(_.toList).view.force @@ -346,7 +374,7 @@ class Executor(executorContext: ExecutorContext, userConf : UserConfig, launcher case query: QueryExecutorConfig => sender ! ExecutorConfig(ClusterConfig.filterOutDefaultConfig(systemConfig)) case HealthCheck => - context.system.scheduler.scheduleOnce(3 second)(HealthCheck) + context.system.scheduler.scheduleOnce(3.second)(HealthCheck) if (state != State.ACTIVE && (transitionEnd - transitionStart) > transitWarningThreshold) { LOG.error(s"Executor status: " + state + s", it takes too long(${transitionEnd - transitionStart}) to do transition") @@ -392,17 +420,18 @@ object Executor { } /** - * when the new DAG is successfully deployed, then we should remove obsolete TaskArgument of old DAG. + * When the new DAG is successfully deployed, then we should remove obsolete + * TaskArgument of old DAG. */ - def removeObsoleteVersion: Unit = { - store = store.map{ kv => + def removeObsoleteVersion(): Unit = { + store = store.map { kv => val (k, list) = kv (k, list.take(1)) } } def removeNewerVersion(currentVersion: Int): Unit = { - store = store.map{ kv => + store = store.map { kv => val (k, list) = kv (k, list.filter(_.dagVersion <= currentVersion)) } @@ -412,18 +441,20 @@ object Executor { case class TaskStopped(task: ActorRef) case class ExecutorSummary( - id: Int, - workerId: WorkerId, - actorPath: String, - logFile: String, - status: String, - taskCount: Int, - tasks: Map[ProcessorId, List[TaskId]], - jvmName: String + id: Int, + workerId: WorkerId, + actorPath: String, + logFile: String, + status: String, + taskCount: Int, + tasks: Map[ProcessorId, List[TaskId]], + jvmName: String ) object ExecutorSummary { - def empty: ExecutorSummary = ExecutorSummary(0, WorkerId.unspecified, "", "", "", 1, null, jvmName = "") + def empty: ExecutorSummary = { + ExecutorSummary(0, WorkerId.unspecified, "", "", "", 1, null, jvmName = "") + } } case class GetExecutorSummary(executorId: Int) diff --git a/streaming/src/main/scala/io/gearpump/streaming/executor/ExecutorRestartPolicy.scala b/streaming/src/main/scala/io/gearpump/streaming/executor/ExecutorRestartPolicy.scala index e485d56df..c40aa5f62 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/executor/ExecutorRestartPolicy.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/executor/ExecutorRestartPolicy.scala @@ -15,18 +15,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.gearpump.streaming.executor -import io.gearpump.streaming.task.TaskId -import io.gearpump.util.RestartPolicy +package io.gearpump.streaming.executor import scala.collection.immutable import scala.concurrent.duration.Duration +import io.gearpump.streaming.task.TaskId +import io.gearpump.util.RestartPolicy + /** - * @param maxNrOfRetries the number of times a executor is allowed to be restarted, negative value means no limit, - * if the limit is exceeded the policy will not allow to restart the executor - * @param withinTimeRange duration of the time window for maxNrOfRetries, Duration.Inf means no window + * + * Controls how many retries to recover failed executors. + * + * @param maxNrOfRetries the number of times a executor is allowed to be restarted, + * negative value means no limit, if the limit is exceeded the policy + * will not allow to restart the executor + * @param withinTimeRange duration of the time window for maxNrOfRetries, Duration.Inf + * means no window */ class ExecutorRestartPolicy(maxNrOfRetries: Int, withinTimeRange: Duration) { private var executorToTaskIds = Map.empty[Int, Set[TaskId]] @@ -45,12 +51,14 @@ class ExecutorRestartPolicy(maxNrOfRetries: Int, withinTimeRange: Duration) { executorToTaskIds.get(executorId).map { taskIds => taskIds.foreach { taskId => taskRestartPolocies.get(taskId).map { policy => - if(!policy.allowRestart) { + if (!policy.allowRestart) { + // scalastyle:off return return false + // scalastyle:on return } } } } true } -} +} \ No newline at end of file diff --git a/streaming/src/main/scala/io/gearpump/streaming/executor/TaskLauncher.scala b/streaming/src/main/scala/io/gearpump/streaming/executor/TaskLauncher.scala index b74288947..d377ea42b 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/executor/TaskLauncher.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/executor/TaskLauncher.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,10 +16,10 @@ * limitations under the License. */ - package io.gearpump.streaming.executor import akka.actor.{Actor, ActorRef, ActorRefFactory, Props} + import io.gearpump.cluster.{ExecutorContext, UserConfig} import io.gearpump.serializer.SerializationFramework import io.gearpump.streaming.ProcessorDescription @@ -28,8 +28,11 @@ import io.gearpump.streaming.task._ import io.gearpump.streaming.util.ActorPathUtil trait ITaskLauncher { + + /** Launch a list of task actors */ def launch(taskIds: List[TaskId], argument: TaskArgument, - context: ActorRefFactory, serializer: SerializationFramework, dispatcher: String): Map[TaskId, ActorRef] + context: ActorRefFactory, serializer: SerializationFramework, dispatcher: String) + : Map[TaskId, ActorRef] } class TaskLauncher( @@ -41,8 +44,10 @@ class TaskLauncher( taskActorClass: Class[_ <: Actor]) extends ITaskLauncher{ - override def launch(taskIds: List[TaskId], argument: TaskArgument, - context: ActorRefFactory, serializer: SerializationFramework, dispatcher: String): Map[TaskId, ActorRef] = { + override def launch( + taskIds: List[TaskId], argument: TaskArgument, + context: ActorRefFactory, serializer: SerializationFramework, dispatcher: String) + : Map[TaskId, ActorRef] = { import argument.{processorDescription, subscribers} val taskConf = userConf.withConfig(processorDescription.taskConf) @@ -57,8 +62,8 @@ class TaskLauncher( var tasks = Map.empty[TaskId, ActorRef] taskIds.foreach { taskId => val task = new TaskWrapper(taskId, taskClass, taskContext, taskConf) - val taskActor = context.actorOf(Props(taskActorClass, taskId, taskContext, userConf, task, serializer). - withDispatcher(dispatcher), ActorPathUtil.taskActorName(taskId)) + val taskActor = context.actorOf(Props(taskActorClass, taskId, taskContext, userConf, task, + serializer).withDispatcher(dispatcher), ActorPathUtil.taskActorName(taskId)) tasks += taskId -> taskActor } tasks @@ -67,7 +72,9 @@ class TaskLauncher( object TaskLauncher { - case class TaskArgument(dagVersion: Int, processorDescription: ProcessorDescription, subscribers: List[Subscriber]) + case class TaskArgument( + dagVersion: Int, processorDescription: ProcessorDescription, + subscribers: List[Subscriber]) def apply(executorContext: ExecutorContext, userConf: UserConfig): TaskLauncher = { import executorContext.{appId, appMaster, appName, executorId} diff --git a/streaming/src/main/scala/io/gearpump/streaming/metrics/ProcessorAggregator.scala b/streaming/src/main/scala/io/gearpump/streaming/metrics/ProcessorAggregator.scala index f92970c99..8be2e72b7 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/metrics/ProcessorAggregator.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/metrics/ProcessorAggregator.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,6 +21,7 @@ package io.gearpump.streaming.metrics import java.util import com.typesafe.config.Config + import io.gearpump.TimeStamp import io.gearpump.cluster.ClientToMaster.ReadOption import io.gearpump.cluster.MasterToClient.HistoryMetricsItem @@ -32,24 +33,20 @@ import io.gearpump.util.HistoryMetricsService.HistoryMetricsConfig /** * - * [[ProcessorAggregator]] does aggregation after grouping by these three attributes: - * 1. processorId - * 2. time section(represented as a index integer) - * 3. metricName(like sendThroughput) - * - * - * It assumes for each [[HistoryMetricsItem]], the name follow the format - * app[appId].processor[processorId].task[taskId].[metricName] + * Does aggregation on metrics after grouping by these three attributes: + * 1. processorId + * 2. time section(represented as a index integer) + * 3. metricName(like sendThroughput) * + * It assumes that for each [[io.gearpump.cluster.MasterToClient.HistoryMetricsItem]], the name + * follow the format app(appId).processor(processorId).task(taskId).(metricName) * * It parses the name to get processorId and metricName. If the parsing fails, then current - * [[HistoryMetricsItem]] will be skipped. - * - * - * This class is optimized for performance. + * [[io.gearpump.cluster.MasterToClient.HistoryMetricsItem]] will be skipped. * + * NOTE: this class is optimized for performance. */ -class ProcessorAggregator(historyMetricConfig: HistoryMetricsConfig) extends MetricsAggregator{ +class ProcessorAggregator(historyMetricConfig: HistoryMetricsConfig) extends MetricsAggregator { def this(config: Config) = { this(HistoryMetricsConfig(config)) @@ -58,9 +55,8 @@ class ProcessorAggregator(historyMetricConfig: HistoryMetricsConfig) extends Met private val aggregatorFactory: AggregatorFactory = new AggregatorFactory() /** - * Accept options: + * Accepts options: * key: "readOption", value: one of "readLatest", "readRecent", "readHistory" - * */ override def aggregate(options: Map[String, String], inputs: Iterator[HistoryMetricsItem]): List[HistoryMetricsItem] = { @@ -68,14 +64,15 @@ class ProcessorAggregator(historyMetricConfig: HistoryMetricsConfig) extends Met aggregate(readOption, inputs, System.currentTimeMillis()) } - def aggregate(readOption: ReadOption.ReadOption, inputs: Iterator[HistoryMetricsItem], now: TimeStamp): - List[HistoryMetricsItem] = { + def aggregate( + readOption: ReadOption.ReadOption, inputs: Iterator[HistoryMetricsItem], now: TimeStamp) + : List[HistoryMetricsItem] = { val (start, end, interval) = getTimeRange(readOption, now) val timeSlotsCount = ((end - start - 1) / interval + 1).toInt val map = new MultiLayerMap[Aggregator](timeSlotsCount) val taskIdentity = new TaskIdentity(0, null) - while(inputs.hasNext) { + while (inputs.hasNext) { val item = inputs.next() if (item.value.isInstanceOf[Meter] || item.value.isInstanceOf[Histogram]) { @@ -97,7 +94,7 @@ class ProcessorAggregator(historyMetricConfig: HistoryMetricsConfig) extends Met val result = new Array[HistoryMetricsItem](map.size) val iterator = map.valueIterator var index = 0 - while(iterator.hasNext()) { + while (iterator.hasNext()) { val op = iterator.next() result(index) = op.result index += 1 @@ -106,8 +103,9 @@ class ProcessorAggregator(historyMetricConfig: HistoryMetricsConfig) extends Met result.toList } - // return (start, end, interval) - private def getTimeRange(readOption: ReadOption.ReadOption, now: TimeStamp): (TimeStamp, TimeStamp, TimeStamp) = { + // Returns (start, end, interval) + private def getTimeRange(readOption: ReadOption.ReadOption, now: TimeStamp) + : (TimeStamp, TimeStamp, TimeStamp) = { readOption match { case ReadOption.ReadRecent => val end = now @@ -120,7 +118,7 @@ class ProcessorAggregator(historyMetricConfig: HistoryMetricsConfig) extends Met val interval = historyMetricConfig.retainHistoryDataIntervalMs (floor(start, interval), floor(end, interval), interval) case _ => - // all data points are aggregated together. + // All data points are aggregated together. (0L, Long.MaxValue, Long.MaxValue) } } @@ -132,7 +130,7 @@ class ProcessorAggregator(historyMetricConfig: HistoryMetricsConfig) extends Met (value / interval) * interval } - // returns "app0.processor0:sendThroughput" as the group Id. + // Returns "app0.processor0:sendThroughput" as the group Id. private def parseName(name: String, result: TaskIdentity): Boolean = { val taskIndex = name.indexOf(TASK_TAG) if (taskIndex > 0) { @@ -157,20 +155,16 @@ object ProcessorAggregator { private val TASK_TAG = ".task" - private class TaskIdentity(var task: Short, var group: String) - /** * * MultiLayerMap has multiple layers. For each layer, there * is a hashMap. * - * To access a value with [[get]], user need to specify first layer Id, then key. - * + * To access a value with get, user need to specify first layer Id, then key. * * This class is optimized for performance. - * */ class MultiLayerMap[Value](layers: Int) { @@ -197,11 +191,11 @@ object ProcessorAggregator { } def size: Int = _size - + def valueIterator: util.Iterator[Value] = { val iterators = new Array[util.Iterator[Value]](layers) var layer = 0 - while(layer < layers) { + while (layer < layers) { iterators(layer) = map(layer).values().iterator() layer += 1 } @@ -213,7 +207,7 @@ object ProcessorAggregator { val map = new Array[java.util.HashMap[String, Value]](layers) var index = 0 val length = map.length - while(index < length) { + while (index < length) { map(index) = new java.util.HashMap[String, Value]() index += 1 } @@ -291,11 +285,11 @@ object ProcessorAggregator { override def result: HistoryMetricsItem = { HistoryMetricsItem(startTime, Meter(name, count, meanRate, - m1, rateUnit)) + m1, rateUnit)) } } - class AggregatorFactory{ + class AggregatorFactory { def create(item: HistoryMetricsItem, name: String): Aggregator = { item.value match { case meter: Meter => new MeterAggregator(name) diff --git a/streaming/src/main/scala/io/gearpump/streaming/metrics/TaskFilterAggregator.scala b/streaming/src/main/scala/io/gearpump/streaming/metrics/TaskFilterAggregator.scala index a9bc167bb..dbf79ecf3 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/metrics/TaskFilterAggregator.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/metrics/TaskFilterAggregator.scala @@ -1,28 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.gearpump.streaming.metrics +import scala.collection.mutable.ListBuffer +import scala.util.{Failure, Success, Try} + import com.typesafe.config.Config + import io.gearpump.cluster.ClientToMaster.ReadOption import io.gearpump.cluster.MasterToClient.HistoryMetricsItem import io.gearpump.metrics.MetricsAggregator -import io.gearpump.util.{LogUtil, Constants} - -import scala.collection.mutable.ListBuffer -import scala.util.{Failure, Success, Try} +import io.gearpump.util.{Constants, LogUtil} /** - * It filters the latest metrics data by specifying a + * Filters the latest metrics data by specifying a * processor Id range, and taskId range. */ -class TaskFilterAggregator (maxLimit: Int) extends MetricsAggregator { +class TaskFilterAggregator(maxLimit: Int) extends MetricsAggregator { - import TaskFilterAggregator._ + import io.gearpump.streaming.metrics.TaskFilterAggregator._ def this(config: Config) = { this(config.getInt(Constants.GEARPUMP_METRICS_MAX_LIMIT)) } - override def aggregate(options: Map[String, String], inputs: Iterator[HistoryMetricsItem]): List[HistoryMetricsItem] = { + override def aggregate(options: Map[String, String], inputs: Iterator[HistoryMetricsItem]) + : List[HistoryMetricsItem] = { + if (options.get(ReadOption.Key) != Some(ReadOption.ReadLatest)) { - // return empty set + // Returns empty set List.empty[HistoryMetricsItem] } else { val parsed = Options.parse(options) @@ -34,7 +55,8 @@ class TaskFilterAggregator (maxLimit: Int) extends MetricsAggregator { } } - def aggregate(options: Options, inputs: Iterator[HistoryMetricsItem]): List[HistoryMetricsItem] = { + def aggregate(options: Options, inputs: Iterator[HistoryMetricsItem]) + : List[HistoryMetricsItem] = { val result = new ListBuffer[HistoryMetricsItem] val effectiveLimit = Math.min(options.limit, maxLimit) @@ -42,7 +64,7 @@ class TaskFilterAggregator (maxLimit: Int) extends MetricsAggregator { val taskIdentity = new TaskIdentity(0, 0) - while(inputs.hasNext && count < effectiveLimit) { + while (inputs.hasNext && count < effectiveLimit) { val item = inputs.next() if (parseName(item.value.name, taskIdentity)) { if (taskIdentity.processor >= options.startProcessor && @@ -57,15 +79,17 @@ class TaskFilterAggregator (maxLimit: Int) extends MetricsAggregator { result.toList } - // Assume the name format is: "app0.processor0.task0:sendThroughput" - // return (processorId, taskId) - // return true if success + // Assume the name format is: "app0.processor0.task0:sendThroughput", returns + // (processorId, taskId) + // + // returns true if success private def parseName(name: String, result: TaskIdentity): Boolean = { val processorStart = name.indexOf(PROCESSOR_TAG) if (processorStart != -1) { val taskStart = name.indexOf(TASK_TAG, processorStart + 1) if (taskStart != -1) { - val processorId = name.substring(processorStart, taskStart).substring(PROCESSOR_TAG.length).toInt + val processorId = name.substring(processorStart, taskStart).substring(PROCESSOR_TAG.length) + .toInt result.processor = processorId val taskEnd = name.indexOf(":", taskStart + 1) if (taskEnd != -1) { @@ -84,7 +108,7 @@ class TaskFilterAggregator (maxLimit: Int) extends MetricsAggregator { } } -object TaskFilterAggregator{ +object TaskFilterAggregator { val StartTask = "startTask" val EndTask = "endTask" val StartProcessor = "startProcessor" @@ -96,7 +120,8 @@ object TaskFilterAggregator{ private class TaskIdentity(var processor: Int, var task: Int) - case class Options(limit: Int, startTask: Int, endTask: Int, startProcessor: Int, endProcessor: Int) + case class Options( + limit: Int, startTask: Int, endTask: Int, startProcessor: Int, endProcessor: Int) private val LOG = LogUtil.getLogger(getClass) @@ -107,7 +132,7 @@ object TaskFilterAggregator{ } def parse(options: Map[String, String]): Options = { - //do sanity check + // Do sanity check val optionTry = Try { val startTask = options.get(StartTask).map(_.toInt).getOrElse(0) val endTask = options.get(EndTask).map(_.toInt).getOrElse(Integer.MAX_VALUE) @@ -120,7 +145,8 @@ object TaskFilterAggregator{ optionTry match { case Success(options) => options case Failure(ex) => - LOG.error("Failed to parse the options in TaskFilterAggregator. Error msg: " + ex.getMessage) + LOG.error("Failed to parse the options in TaskFilterAggregator. Error msg: " + + ex.getMessage) null } } diff --git a/streaming/src/main/scala/io/gearpump/streaming/package.scala b/streaming/src/main/scala/io/gearpump/streaming/package.scala index d2555227b..95d51f046 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/package.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/package.scala @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.gearpump package object streaming { diff --git a/streaming/src/main/scala/io/gearpump/streaming/sink/DataSink.scala b/streaming/src/main/scala/io/gearpump/streaming/sink/DataSink.scala index 3e062edbc..b03661995 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/sink/DataSink.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/sink/DataSink.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -22,12 +22,10 @@ import io.gearpump.Message import io.gearpump.streaming.task.TaskContext /** - * interface to implement custom data sink - * where result of a DAG is typically written - * + * Interface to implement custom data sink where result of a DAG is typically written * a DataSink could be a data store like HBase or simply a console * - * an example would be like + * An example would be like: * {{{ * class ConsoleSink extends DataSink[String] { * @@ -41,26 +39,26 @@ import io.gearpump.streaming.task.TaskContext * } * }}} * - * subclass is required to be serializable + * Subclass is required to be serializable */ trait DataSink extends java.io.Serializable { /** - * open connection to data sink + * Opens connection to data sink * invoked at onStart() method of [[io.gearpump.streaming.task.Task]] * @param context is the task context at runtime */ def open(context: TaskContext): Unit /** - * write message into data sink + * Writes message into data sink * invoked at onNext() method of [[io.gearpump.streaming.task.Task]] * @param message wraps data to be written out */ def write(message: Message): Unit /** - * close connection to data sink + * Closes connection to data sink * invoked at onClose() method of [[io.gearpump.streaming.task.Task]] */ def close(): Unit diff --git a/streaming/src/main/scala/io/gearpump/streaming/sink/DataSinkProcessor.scala b/streaming/src/main/scala/io/gearpump/streaming/sink/DataSinkProcessor.scala index e83b40d93..d753cc2bd 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/sink/DataSinkProcessor.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/sink/DataSinkProcessor.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,11 +19,12 @@ package io.gearpump.streaming.sink import akka.actor.ActorSystem -import io.gearpump.streaming.Processor + import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.Processor /** - * utility that helps user to create a DAG ending in [[DataSink]] + * Utility that helps user to create a DAG ending in [[DataSink]] * user should pass in a [[DataSink]]. * * here is an example to build a DAG that does word count and write to KafkaSink @@ -36,10 +37,12 @@ import io.gearpump.cluster.UserConfig * }}} */ object DataSinkProcessor { - def apply(dataSink: DataSink, - parallelism: Int, - description: String = "", - taskConf: UserConfig = UserConfig.empty)(implicit system: ActorSystem): Processor[DataSinkTask] = { + def apply( + dataSink: DataSink, + parallelism: Int, + description: String = "", + taskConf: UserConfig = UserConfig.empty)(implicit system: ActorSystem) + : Processor[DataSinkTask] = { Processor[DataSinkTask](parallelism, description = description, taskConf.withValue[DataSink](DataSinkTask.DATA_SINK, dataSink)) } diff --git a/streaming/src/main/scala/io/gearpump/streaming/sink/DataSinkTask.scala b/streaming/src/main/scala/io/gearpump/streaming/sink/DataSinkTask.scala index 95a671844..7436617fc 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/sink/DataSinkTask.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/sink/DataSinkTask.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,19 +18,19 @@ package io.gearpump.streaming.sink -import io.gearpump.streaming.task.{Task, TaskContext, StartTime} import io.gearpump.Message import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.task.{StartTime, Task, TaskContext} object DataSinkTask { val DATA_SINK = "data_sink" } /** - * general task that runs any [[DataSink]] + * General task that runs any [[DataSink]] */ -class DataSinkTask (context: TaskContext, conf: UserConfig) extends Task(context, conf) { - import DataSinkTask._ +class DataSinkTask(context: TaskContext, conf: UserConfig) extends Task(context, conf) { + import io.gearpump.streaming.sink.DataSinkTask._ private val sink = conf.getValue[DataSink](DATA_SINK).get @@ -47,5 +47,4 @@ class DataSinkTask (context: TaskContext, conf: UserConfig) extends Task(context LOG.info("closing data sink...") sink.close() } - } diff --git a/streaming/src/main/scala/io/gearpump/streaming/source/DataSource.scala b/streaming/src/main/scala/io/gearpump/streaming/source/DataSource.scala index bee81fb47..9cb7ca080 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/source/DataSource.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/source/DataSource.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,13 +19,13 @@ package io.gearpump.streaming.source import io.gearpump.streaming.task.TaskContext -import io.gearpump.{TimeStamp, Message} +import io.gearpump.{Message, TimeStamp} /** - * interface to implement custom source where data is read into the system. + * Interface to implement custom source where data is read into the system. * a DataSource could be a message queue like kafka or simply data generation source. * - * an example would be like + * An example would be like * {{{ * GenStringSource extends DataSource { * @@ -44,7 +44,7 @@ import io.gearpump.{TimeStamp, Message} trait DataSource extends java.io.Serializable { /** - * open connection to data source + * Opens connection to data source * invoked in onStart() method of [[io.gearpump.streaming.source.DataSourceTask]] * @param context is the task context at runtime * @param startTime is the start time of system @@ -52,7 +52,7 @@ trait DataSource extends java.io.Serializable { def open(context: TaskContext, startTime: Option[TimeStamp]): Unit /** - * read a number of messages from data source. + * Reads a number of messages from data source. * invoked in each onNext() method of [[io.gearpump.streaming.source.DataSourceTask]] * @param batchSize max number of messages to read * @return a list of messages wrapped in [[io.gearpump.Message]] @@ -60,7 +60,7 @@ trait DataSource extends java.io.Serializable { def read(batchSize: Int): List[Message] /** - * close connection to data source. + * Closes connection to data source. * invoked in onStop() method of [[io.gearpump.streaming.source.DataSourceTask]] */ def close(): Unit diff --git a/streaming/src/main/scala/io/gearpump/streaming/source/DataSourceConfig.scala b/streaming/src/main/scala/io/gearpump/streaming/source/DataSourceConfig.scala index e7c0599f5..6ca939f1a 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/source/DataSourceConfig.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/source/DataSourceConfig.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -22,5 +22,4 @@ object DataSourceConfig { val SOURCE_READ_BATCH_SIZE = "gearpump.source.read.batch.size" val SOURCE_TIMESTAMP_FILTER = "gearpump.source.timestamp.filter.class" - } diff --git a/streaming/src/main/scala/io/gearpump/streaming/source/DataSourceProcessor.scala b/streaming/src/main/scala/io/gearpump/streaming/source/DataSourceProcessor.scala index 61eabcc21..384b86a95 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/source/DataSourceProcessor.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/source/DataSourceProcessor.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,14 +19,15 @@ package io.gearpump.streaming.source import akka.actor.ActorSystem -import io.gearpump.streaming.Processor + import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.Processor /** - * utility that helps user to create a DAG starting with [[DataSourceTask]] + * Utility that helps user to create a DAG starting with [[DataSourceTask]] * user should pass in a [[DataSource]] * - * here is an example to build a DAG that reads from Kafka source followed by word count + * Here is an example to build a DAG that reads from Kafka source followed by word count * {{{ * val source = new KafkaSource() * val sourceProcessor = DataSourceProcessor(source, 1) @@ -36,10 +37,12 @@ import io.gearpump.cluster.UserConfig * }}} */ object DataSourceProcessor { - def apply(dataSource: DataSource, - parallelism: Int, - description: String = "", - taskConf: UserConfig = UserConfig.empty)(implicit system: ActorSystem): Processor[DataSourceTask] = { + def apply( + dataSource: DataSource, + parallelism: Int, + description: String = "", + taskConf: UserConfig = UserConfig.empty)(implicit system: ActorSystem) + : Processor[DataSourceTask] = { Processor[DataSourceTask](parallelism, description = description, taskConf.withValue[DataSource](DataSourceTask.DATA_SOURCE, dataSource)) } diff --git a/streaming/src/main/scala/io/gearpump/streaming/source/DataSourceTask.scala b/streaming/src/main/scala/io/gearpump/streaming/source/DataSourceTask.scala index 4c65100fc..d9b211034 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/source/DataSourceTask.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/source/DataSourceTask.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,26 +18,27 @@ package io.gearpump.streaming.source -import io.gearpump.streaming.task.{Task, StartTime, TaskContext} import io.gearpump._ import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.task.{StartTime, Task, TaskContext} object DataSourceTask { val DATA_SOURCE = "data_source" } /** - * general task that runs any [[DataSource]] - * see [[DataSourceProcessor]] for its usage + * Task container for [[io.gearpump.streaming.source.DataSource]]. + * See [[io.gearpump.streaming.source.DataSourceProcessor]] for its usage * - * DataSourceTask calls - * - `DataSource.open` in `onStart` and pass in [[io.gearpump.streaming.task.TaskContext]] and application start time - * - `DataSource.read` in each `onNext`, which reads a batch of messages whose size are defined by - * `gearpump.source.read.batch.size`. - * - `DataSource.close` in `onStop` + * DataSourceTask calls: + * - `DataSource.open()` in `onStart` and pass in [[io.gearpump.streaming.task.TaskContext]] + * and application start time + * - `DataSource.read()` in each `onNext`, which reads a batch of messages whose size are + * defined by `gearpump.source.read.batch.size`. + * - `DataSource.close()` in `onStop` */ class DataSourceTask(context: TaskContext, conf: UserConfig) extends Task(context, conf) { - import DataSourceTask._ + import io.gearpump.streaming.source.DataSourceTask._ private val source = conf.getValue[DataSource](DATA_SOURCE).get private val batchSize = conf.getInt(DataSourceConfig.SOURCE_READ_BATCH_SIZE).getOrElse(1000) diff --git a/streaming/src/main/scala/io/gearpump/streaming/source/DefaultTimeStampFilter.scala b/streaming/src/main/scala/io/gearpump/streaming/source/DefaultTimeStampFilter.scala index a9ee6745c..b7d4e903b 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/source/DefaultTimeStampFilter.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/source/DefaultTimeStampFilter.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,10 +19,10 @@ package io.gearpump.streaming.source import io.gearpump.streaming.transaction.api.TimeStampFilter -import io.gearpump.{TimeStamp, Message} +import io.gearpump.{Message, TimeStamp} /** - * default TimeStampFilter that filters out messages with smaller timestamps + * TimeStampFilter filters out messages which have obsolete (smaller) timestamp. */ class DefaultTimeStampFilter extends TimeStampFilter { override def filter(msg: Message, predicate: TimeStamp): Option[Message] = { diff --git a/streaming/src/main/scala/io/gearpump/streaming/state/api/Monoid.scala b/streaming/src/main/scala/io/gearpump/streaming/state/api/Monoid.scala index a25e20e6f..dfe3e9309 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/state/api/Monoid.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/state/api/Monoid.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,10 +19,10 @@ package io.gearpump.streaming.state.api trait Monoid[T] extends java.io.Serializable { - def plus(l: T, r: T): T - def zero: T + def plus(l: T, r: T): T + def zero: T } trait Group[T] extends Monoid[T] { - def minus(l: T, r: T): T - } \ No newline at end of file + def minus(l: T, r: T): T +} \ No newline at end of file diff --git a/streaming/src/main/scala/io/gearpump/streaming/state/api/MonoidState.scala b/streaming/src/main/scala/io/gearpump/streaming/state/api/MonoidState.scala index dadbba6b4..238eab4eb 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/state/api/MonoidState.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/state/api/MonoidState.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -28,9 +28,9 @@ import io.gearpump.TimeStamp * the incoming value using monoid.plus to get a new state value */ abstract class MonoidState[T](monoid: Monoid[T]) extends PersistentState[T] { - // left state updated by messages before checkpoint time + // Left state updated by messages before checkpoint time private[state] var left: T = monoid.zero - // right state updated by message after checkpoint time + // Right state updated by message after checkpoint time private[state] var right: T = monoid.zero protected var checkpointTime = Long.MaxValue diff --git a/streaming/src/main/scala/io/gearpump/streaming/state/api/PersistentState.scala b/streaming/src/main/scala/io/gearpump/streaming/state/api/PersistentState.scala index 6c595da4e..f1b923ad8 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/state/api/PersistentState.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/state/api/PersistentState.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -30,33 +30,32 @@ import io.gearpump._ trait PersistentState[T] { /** - * recover state to a previous checkpoint + * Recovers state to a previous checkpoint * usually invoked by the framework */ def recover(timestamp: TimeStamp, bytes: Array[Byte]): Unit /** - * update state on a new message + * Updates state on a new message * this is invoked by user */ def update(timestamp: TimeStamp, t: T): Unit /** - * set next checkpoint time + * Sets next checkpoint time * should be invoked by the framework */ def setNextCheckpointTime(timeStamp: TimeStamp): Unit /** - * get a binary snapshot of state + * Gets a binary snapshot of state * usually invoked by the framework */ def checkpoint(): Array[Byte] /** - * unwrap the raw value of state + * Unwraps the raw value of state */ def get: Option[T] } - diff --git a/streaming/src/main/scala/io/gearpump/streaming/state/api/PersistentTask.scala b/streaming/src/main/scala/io/gearpump/streaming/state/api/PersistentTask.scala index fbf507f88..a8a5d5d3f 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/state/api/PersistentTask.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/state/api/PersistentTask.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,16 +19,15 @@ package io.gearpump.streaming.state.api import java.util.concurrent.TimeUnit +import scala.concurrent.duration.FiniteDuration import io.gearpump.cluster.UserConfig -import io.gearpump.streaming.state.impl.{PersistentStateConfig, CheckpointManager} +import io.gearpump.streaming.state.impl.{CheckpointManager, PersistentStateConfig} import io.gearpump.streaming.task.{ReportCheckpointClock, StartTime, Task, TaskContext} import io.gearpump.streaming.transaction.api.CheckpointStoreFactory import io.gearpump.util.LogUtil import io.gearpump.{Message, TimeStamp} -import scala.concurrent.duration.FiniteDuration - object PersistentTask { val CHECKPOINT = Message("checkpoint") val LOG = LogUtil.getLogger(getClass) @@ -42,35 +41,32 @@ object PersistentTask { */ abstract class PersistentTask[T](taskContext: TaskContext, conf: UserConfig) extends Task(taskContext, conf) { - import io.gearpump.streaming.state.api.PersistentTask._ import taskContext._ - val checkpointStoreFactory = conf.getValue[CheckpointStoreFactory](PersistentStateConfig.STATE_CHECKPOINT_STORE_FACTORY).get + import io.gearpump.streaming.state.api.PersistentTask._ + + val checkpointStoreFactory = conf.getValue[CheckpointStoreFactory]( + PersistentStateConfig.STATE_CHECKPOINT_STORE_FACTORY).get val checkpointStore = checkpointStoreFactory.getCheckpointStore(conf, taskContext) val checkpointInterval = conf.getLong(PersistentStateConfig.STATE_CHECKPOINT_INTERVAL_MS).get val checkpointManager = new CheckpointManager(checkpointInterval, checkpointStore) - // system time interval to attempt checkpoint + // System time interval to attempt checkpoint private val checkpointAttemptInterval = 1000L /** - * subclass should override this method to pass in - * a PersistentState - * - * the framework has already offered two states - * - * - NonWindowState - * state with no time or other boundary - * - WindowState - * each state is bounded by a time window + * Subclass should override this method to pass in a PersistentState. the framework has already + * offered two states: + * - NonWindowState: state with no time or other boundary + * - WindowState: each state is bounded by a time window */ def persistentState: PersistentState[T] /** - * subclass should override this method to specify how a - * new message should update state + * Subclass should override this method to specify how a new message should update state */ def processMessage(state: PersistentState[T], message: Message): Unit + /** Persistent state that will be stored (by checkpointing) automatically to storage like HDFS */ val state = persistentState final override def onStart(startTime: StartTime): Unit = { diff --git a/streaming/src/main/scala/io/gearpump/streaming/state/api/Serializer.scala b/streaming/src/main/scala/io/gearpump/streaming/state/api/Serializer.scala index 4b8e2f5dc..f87b2249a 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/state/api/Serializer.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/state/api/Serializer.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,6 +21,6 @@ package io.gearpump.streaming.state.api import scala.util.Try trait Serializer[T] extends java.io.Serializable { - def serialize(t: T): Array[Byte] - def deserialize(bytes: Array[Byte]): Try[T] - } + def serialize(t: T): Array[Byte] + def deserialize(bytes: Array[Byte]): Try[T] +} \ No newline at end of file diff --git a/streaming/src/main/scala/io/gearpump/streaming/state/impl/CheckpointManager.scala b/streaming/src/main/scala/io/gearpump/streaming/state/impl/CheckpointManager.scala index 5fcbbcbcc..76c91c713 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/state/impl/CheckpointManager.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/state/impl/CheckpointManager.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,6 +21,7 @@ package io.gearpump.streaming.state.impl import io.gearpump.TimeStamp import io.gearpump.streaming.transaction.api.CheckpointStore +/** Manage physical checkpoints to persitent storage like HDFS */ class CheckpointManager(checkpointInterval: Long, checkpointStore: CheckpointStore) { @@ -34,7 +35,7 @@ class CheckpointManager(checkpointInterval: Long, def checkpoint(timestamp: TimeStamp, checkpoint: Array[Byte]): Option[TimeStamp] = { checkpointStore.persist(timestamp, checkpoint) checkpointTime = checkpointTime.collect { case time if maxMessageTime > time => - time + (1 + (maxMessageTime - time) / checkpointInterval) * checkpointInterval + time + (1 + (maxMessageTime - time) / checkpointInterval) * checkpointInterval } checkpointTime diff --git a/streaming/src/main/scala/io/gearpump/streaming/state/impl/InMemoryCheckpointStore.scala b/streaming/src/main/scala/io/gearpump/streaming/state/impl/InMemoryCheckpointStore.scala index 164b93298..21623e374 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/state/impl/InMemoryCheckpointStore.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/state/impl/InMemoryCheckpointStore.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,7 +21,7 @@ package io.gearpump.streaming.state.impl import io.gearpump.TimeStamp import io.gearpump.cluster.UserConfig import io.gearpump.streaming.task.TaskContext -import io.gearpump.streaming.transaction.api.{CheckpointStoreFactory, CheckpointStore} +import io.gearpump.streaming.transaction.api.{CheckpointStore, CheckpointStoreFactory} /** * an in memory store provided for test @@ -41,7 +41,6 @@ class InMemoryCheckpointStore extends CheckpointStore { override def close(): Unit = { checkpoints = Map.empty[TimeStamp, Array[Byte]] } - } class InMemoryCheckpointStoreFactory extends CheckpointStoreFactory { diff --git a/streaming/src/main/scala/io/gearpump/streaming/state/impl/NonWindowState.scala b/streaming/src/main/scala/io/gearpump/streaming/state/impl/NonWindowState.scala index aefd7e1f7..dcd391818 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/state/impl/NonWindowState.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/state/impl/NonWindowState.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,20 +18,22 @@ package io.gearpump.streaming.state.impl +import org.slf4j.Logger + import io.gearpump.TimeStamp import io.gearpump.streaming.state.api.{Monoid, MonoidState, Serializer} -import io.gearpump.util.LogUtil import io.gearpump.streaming.state.impl.NonWindowState._ -import org.slf4j.Logger +import io.gearpump.util.LogUtil object NonWindowState { val LOG: Logger = LogUtil.getLogger(classOf[NonWindowState[_]]) } /** - * a MonoidState storing non-window state + * a MonoidState storing non-window state */ -class NonWindowState[T](monoid: Monoid[T], serializer: Serializer[T]) extends MonoidState[T](monoid) { +class NonWindowState[T](monoid: Monoid[T], serializer: Serializer[T]) + extends MonoidState[T](monoid) { override def recover(timestamp: TimeStamp, bytes: Array[Byte]): Unit = { serializer.deserialize(bytes).foreach(left = _) diff --git a/streaming/src/main/scala/io/gearpump/streaming/state/impl/PersistentStateConfig.scala b/streaming/src/main/scala/io/gearpump/streaming/state/impl/PersistentStateConfig.scala index ecbb0927a..d7488d780 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/state/impl/PersistentStateConfig.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/state/impl/PersistentStateConfig.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/streaming/src/main/scala/io/gearpump/streaming/state/impl/Window.scala b/streaming/src/main/scala/io/gearpump/streaming/state/impl/Window.scala index 13f0eef8f..63cdf0660 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/state/impl/Window.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/state/impl/Window.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,7 +20,7 @@ package io.gearpump.streaming.state.impl import io.gearpump.TimeStamp /** - * used in window applications + * Used in window applications * it keeps the current window and slide ahead when the window expires */ class Window(val windowSize: Long, val windowStep: Long) { diff --git a/streaming/src/main/scala/io/gearpump/streaming/state/impl/WindowConfig.scala b/streaming/src/main/scala/io/gearpump/streaming/state/impl/WindowConfig.scala index 41395f61b..d7d37768a 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/state/impl/WindowConfig.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/state/impl/WindowConfig.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/streaming/src/main/scala/io/gearpump/streaming/state/impl/WindowState.scala b/streaming/src/main/scala/io/gearpump/streaming/state/impl/WindowState.scala index 007c9a48f..30382c0ed 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/state/impl/WindowState.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/state/impl/WindowState.scala @@ -2,12 +2,12 @@ * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information - * regarding cstateyright ownership. The ASF licenses this file + * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance - * with the License. You may obtain a cstatey of the License at + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,14 +18,15 @@ package io.gearpump.streaming.state.impl +import scala.collection.immutable.TreeMap + +import org.slf4j.Logger + import io.gearpump.TimeStamp -import io.gearpump.streaming.state.api.{Group, Serializer, MonoidState} -import io.gearpump.streaming.task.TaskContext +import io.gearpump.streaming.state.api.{Group, MonoidState, Serializer} import io.gearpump.streaming.state.impl.WindowState._ +import io.gearpump.streaming.task.TaskContext import io.gearpump.util.LogUtil -import org.slf4j.Logger - -import scala.collection.immutable.TreeMap /** * an interval is a dynamic time range that is divided by window boundary and checkpoint time @@ -51,10 +52,9 @@ object WindowState { * possible to undo the update by messages that have left the window */ class WindowState[T](group: Group[T], - serializer: Serializer[TreeMap[Interval, T]], - taskContext: TaskContext, - window: Window) - extends MonoidState[T](group) { + serializer: Serializer[TreeMap[Interval, T]], + taskContext: TaskContext, + window: Window) extends MonoidState[T](group) { /** * each interval has a state updated by message with timestamp in * [interval.startTime, interval.endTime) @@ -67,11 +67,11 @@ class WindowState[T](group: Group[T], window.slideTo(timestamp) serializer.deserialize(bytes) .foreach { states => - intervalStates = states - left = states.foldLeft(left) { case (accum, iter) => - group.plus(accum, iter._2) + intervalStates = states + left = states.foldLeft(left) { case (accum, iter) => + group.plus(accum, iter._2) + } } - } } override def update(timestamp: TimeStamp, t: T): Unit = { @@ -115,14 +115,17 @@ class WindowState[T](group: Group[T], } /** - * each message will update state in corresponding Interval[StartTime, endTime), + * Each message will update state in corresponding Interval[StartTime, endTime), + * * which is decided by the message's timestamp t where - * startTime = Math.max(lowerBound1, lowerBound2, checkpointTime) - * endTime = Math.min(upperBound1, upperBound2, checkpointTime) - * lowerBound1 = step * Nmax1 <= t - * lowerBound2 = step * Nmax2 + size <= t - * upperBound1 = step * Nmin1 > t - * upperBound2 = step * Nmin2 + size > t + * {{{ + * startTime = Math.max(lowerBound1, lowerBound2, checkpointTime) + * endTime = Math.min(upperBound1, upperBound2, checkpointTime) + * lowerBound1 = step * Nmax1 <= t + * lowerBound2 = step * Nmax2 + size <= t + * upperBound1 = step * Nmin1 > t + * upperBound2 = step * Nmin2 + size > t + * }}} */ private[impl] def getInterval(timestamp: TimeStamp, checkpointTime: TimeStamp): Interval = { val windowSize = window.windowSize @@ -144,7 +147,8 @@ class WindowState[T](group: Group[T], } } - private[impl] def updateIntervalStates(timestamp: TimeStamp, t: T, checkpointTime: TimeStamp): Unit = { + private[impl] def updateIntervalStates(timestamp: TimeStamp, t: T, checkpointTime: TimeStamp) + : Unit = { val interval = getInterval(timestamp, checkpointTime) intervalStates.get(interval) match { case Some(st) => @@ -154,8 +158,8 @@ class WindowState[T](group: Group[T], } } - private[impl] def getIntervalStates(startTime: TimeStamp, endTime: TimeStamp): TreeMap[Interval, T] = { + private[impl] def getIntervalStates(startTime: TimeStamp, endTime: TimeStamp) + : TreeMap[Interval, T] = { intervalStates.dropWhile(_._1.endTime <= startTime).takeWhile(_._1.endTime <= endTime) } - } diff --git a/streaming/src/main/scala/io/gearpump/streaming/storage/AppDataStore.scala b/streaming/src/main/scala/io/gearpump/streaming/storage/AppDataStore.scala index dcdc2ffe1..962f48f25 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/storage/AppDataStore.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/storage/AppDataStore.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,12 +21,10 @@ import scala.concurrent._ /** * Generic storage to store KV Data. - * */ trait AppDataStore { def put(key: String, value: Any): Future[Any] - def get(key: String) : Future[Any] + def get(key: String): Future[Any] } - diff --git a/streaming/src/main/scala/io/gearpump/streaming/storage/InMemoryAppStoreOnMaster.scala b/streaming/src/main/scala/io/gearpump/streaming/storage/InMemoryAppStoreOnMaster.scala index 13e3dae0f..f1a19a80e 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/storage/InMemoryAppStoreOnMaster.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/storage/InMemoryAppStoreOnMaster.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,12 +17,13 @@ */ package io.gearpump.streaming.storage +import scala.concurrent.Future + import akka.actor.ActorRef import akka.pattern.ask -import io.gearpump.cluster.AppMasterToMaster.{GetAppDataResult, GetAppData, SaveAppData} -import io.gearpump.util.Constants -import scala.concurrent.Future +import io.gearpump.cluster.AppMasterToMaster.{GetAppData, GetAppDataResult, SaveAppData} +import io.gearpump.util.Constants /** * In memory application storage located on master nodes @@ -36,8 +37,8 @@ class InMemoryAppStoreOnMaster(appId: Int, master: ActorRef) extends AppDataStor } override def get(key: String): Future[Any] = { - master.ask(GetAppData(appId, key)).asInstanceOf[Future[GetAppDataResult]].map{result => - if(result.key.equals(key)) { + master.ask(GetAppData(appId, key)).asInstanceOf[Future[GetAppDataResult]].map { result => + if (result.key.equals(key)) { result.value } else { null diff --git a/streaming/src/main/scala/io/gearpump/streaming/task/ExpressTransport.scala b/streaming/src/main/scala/io/gearpump/streaming/task/ExpressTransport.scala index 2dbdd1483..a3cb6e1e2 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/task/ExpressTransport.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/task/ExpressTransport.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,19 +19,16 @@ package io.gearpump.streaming.task import akka.actor.{ActorRef, ExtendedActorSystem} -import io.gearpump.serializer.SerializationFramework + +import io.gearpump.Message import io.gearpump.transport.netty.TaskMessage import io.gearpump.transport.{Express, HostPort} -import io.gearpump.Message - -import scala.collection.mutable import io.gearpump.util.AkkaHelper /** * ExpressTransport wire the networking function from default akka - * networking to customized implementation [[Express]]. - * - * See [[Express]] for more information. + * networking to customized implementation [[io.gearpump.transport.Express]]. * + * See [[io.gearpump.transport.Express]] for more information. */ trait ExpressTransport { this: TaskActor => @@ -39,15 +36,15 @@ trait ExpressTransport { final val express = Express(context.system) implicit val system = context.system.asInstanceOf[ExtendedActorSystem] - final def local = express.localHost + final def local: HostPort = express.localHost lazy val sourceId = TaskId.toLong(taskId) lazy val sessionRef: ActorRef = { AkkaHelper.actorFor(system, s"/session#$sessionId") } - def transport(msg : AnyRef, remotes : TaskId *): Unit = { - var serializedMessage : AnyRef = null + def transport(msg: AnyRef, remotes: TaskId*): Unit = { + var serializedMessage: AnyRef = null remotes.foreach { remote => val transportId = TaskId.toLong(remote) @@ -69,7 +66,8 @@ trait ExpressTransport { if (remoteAddress.isDefined) { express.transport(taskMessage, remoteAddress.get) } else { - LOG.error(s"Can not find target task $remote, maybe the application is undergoing recovery") + LOG.error( + s"Can not find target task $remote, maybe the application is undergoing recovery") } } } diff --git a/streaming/src/main/scala/io/gearpump/streaming/task/SerializedMessage.scala b/streaming/src/main/scala/io/gearpump/streaming/task/SerializedMessage.scala index ff2ca89c1..9f9bf1b02 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/task/SerializedMessage.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/task/SerializedMessage.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/streaming/src/main/scala/io/gearpump/streaming/task/SerializerResolver.scala b/streaming/src/main/scala/io/gearpump/streaming/task/SerializerResolver.scala index edac7f012..72bc7db6a 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/task/SerializerResolver.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/task/SerializerResolver.scala @@ -15,19 +15,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package io.gearpump.streaming.task -import io.gearpump.esotericsoftware.kryo.util.ObjectMap -import io.gearpump.esotericsoftware.kryo.util.IntMap +import io.gearpump.esotericsoftware.kryo.util.{IntMap, ObjectMap} import io.gearpump.streaming.task.SerializerResolver.Registration private[task] class SerializerResolver { private var classId = 0 - val idToRegistration = new IntMap[Registration]() - val classToRegistration = new ObjectMap[Class[_], Registration]() + private val idToRegistration = new IntMap[Registration]() + private val classToRegistration = new ObjectMap[Class[_], Registration]() - def register(clazz: Class[_], serializer: TaskMessageSerializer[_]): Unit = { - val registration = Registration(classId, clazz, serializer) + def register[T](clazz: Class[T], serializer: TaskMessageSerializer[T]): Unit = { + val registration = new Registration(classId, clazz, serializer) idToRegistration.put(classId, registration) classToRegistration.put(clazz, registration) classId += 1 @@ -42,6 +42,6 @@ private[task] class SerializerResolver { } } -object SerializerResolver{ - case class Registration(id: Int, clazz: Class[_], serializer: TaskMessageSerializer[_]) +object SerializerResolver { + class Registration(val id: Int, val clazz: Class[_], val serializer: TaskMessageSerializer[_]) } diff --git a/streaming/src/main/scala/io/gearpump/streaming/task/StartTime.scala b/streaming/src/main/scala/io/gearpump/streaming/task/StartTime.scala index 75e35214a..6bc8b15e3 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/task/StartTime.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/task/StartTime.scala @@ -1,5 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package io.gearpump.streaming.task import io.gearpump.TimeStamp -case class StartTime(startTime : TimeStamp = 0) +/** Start time of streaming application. All message older than start time will be dropped */ +case class StartTime(startTime: TimeStamp = 0) diff --git a/streaming/src/main/scala/io/gearpump/streaming/task/StreamingTransportSerializer.scala b/streaming/src/main/scala/io/gearpump/streaming/task/StreamingTransportSerializer.scala index 52b802184..17d0b1beb 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/task/StreamingTransportSerializer.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/task/StreamingTransportSerializer.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,12 +19,13 @@ package io.gearpump.streaming.task import java.io.{DataInput, DataOutput} -import io.gearpump.streaming.{LatencyProbeSerializer, InitialAckRequestSerializer, AckRequestSerializer, AckSerializer} +import org.slf4j.Logger + +import io.gearpump.streaming.{AckRequestSerializer, AckSerializer, InitialAckRequestSerializer, LatencyProbeSerializer} import io.gearpump.transport.netty.ITransportMessageSerializer import io.gearpump.util.LogUtil -import org.slf4j.Logger -class StreamingTransportSerializer extends ITransportMessageSerializer{ +class StreamingTransportSerializer extends ITransportMessageSerializer { private val log: Logger = LogUtil.getLogger(getClass) private val serializers = new SerializerResolver @@ -36,7 +37,7 @@ class StreamingTransportSerializer extends ITransportMessageSerializer{ override def serialize(dataOutput: DataOutput, obj: Object): Unit = { val registration = serializers.getRegistration(obj.getClass) - if(registration != null) { + if (registration != null) { dataOutput.writeInt(registration.id) registration.serializer.asInstanceOf[TaskMessageSerializer[AnyRef]].write(dataOutput, obj) } else { @@ -47,7 +48,7 @@ class StreamingTransportSerializer extends ITransportMessageSerializer{ override def deserialize(dataInput: DataInput, length: Int): Object = { val classID = dataInput.readInt() val registration = serializers.getRegistration(classID) - if(registration != null) { + if (registration != null) { registration.serializer.asInstanceOf[TaskMessageSerializer[AnyRef]].read(dataInput) } else { log.error(s"Can not find serializer for class id $classID") @@ -57,7 +58,7 @@ class StreamingTransportSerializer extends ITransportMessageSerializer{ override def getLength(obj: Object): Int = { val registration = serializers.getRegistration(obj.getClass) - if(registration != null) { + if (registration != null) { registration.serializer.asInstanceOf[TaskMessageSerializer[AnyRef]].getLength(obj) + 4 } else { log.error(s"Can not find serializer for class type ${obj.getClass}") diff --git a/streaming/src/main/scala/io/gearpump/streaming/task/Subscriber.scala b/streaming/src/main/scala/io/gearpump/streaming/task/Subscriber.scala index 72d46f775..d20074bb4 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/task/Subscriber.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/task/Subscriber.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,7 +19,7 @@ package io.gearpump.streaming.task import io.gearpump.partitioner.PartitionerDescription -import io.gearpump.streaming.{LifeTime, DAG} +import io.gearpump.streaming.{DAG, LifeTime} /** * Each processor can have multiple downstream subscribers. @@ -30,8 +30,8 @@ import io.gearpump.streaming.{LifeTime, DAG} * @param processorId subscriber processor Id * @param partitionerDescription subscriber partitioner */ - -case class Subscriber(processorId: Int, partitionerDescription: PartitionerDescription, parallelism: Int, lifeTime: LifeTime) +case class Subscriber(processorId: Int, partitionerDescription: PartitionerDescription, + parallelism: Int, lifeTime: LifeTime) object Subscriber { @@ -50,7 +50,8 @@ object Subscriber { edges.foldLeft(List.empty[Subscriber]) { (list, nodeEdgeNode) => val (_, partitioner, downstreamProcessorId) = nodeEdgeNode val downstreamProcessor = dag.processors(downstreamProcessorId) - list :+ Subscriber(downstreamProcessorId, partitioner, downstreamProcessor.parallelism, downstreamProcessor.life) + list :+ Subscriber(downstreamProcessorId, partitioner, + downstreamProcessor.parallelism, downstreamProcessor.life) } } } diff --git a/streaming/src/main/scala/io/gearpump/streaming/task/Subscription.scala b/streaming/src/main/scala/io/gearpump/streaming/task/Subscription.scala index 0b1fa292e..155edf4c8 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/task/Subscription.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/task/Subscription.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,20 +18,22 @@ package io.gearpump.streaming.task +import org.slf4j.Logger + import io.gearpump.google.common.primitives.Shorts -import io.gearpump.partitioner.{Partitioner, MulticastPartitioner, UnicastPartitioner} +import io.gearpump.partitioner.{MulticastPartitioner, Partitioner, UnicastPartitioner} import io.gearpump.streaming.AppMasterToExecutor.MsgLostException import io.gearpump.streaming.LifeTime import io.gearpump.streaming.task.Subscription._ import io.gearpump.util.LogUtil import io.gearpump.{Message, TimeStamp} -import org.slf4j.Logger /** - * This manage the output and message clock for single downstream processor + * Manges the output and message clock for single downstream processor * * @param subscriber downstream processor - * @param maxPendingMessageCount trigger flow control. Should be bigger than maxPendingMessageCountPerAckRequest + * @param maxPendingMessageCount trigger flow control. Should be bigger than + * maxPendingMessageCountPerAckRequest * @param ackOnceEveryMessageCount send on AckRequest to the target */ class Subscription( @@ -44,9 +46,10 @@ class Subscription( ackOnceEveryMessageCount: Int = ONE_ACKREQUEST_EVERY_MESSAGE_COUNT) { assert(maxPendingMessageCount >= ackOnceEveryMessageCount) - assert(maxPendingMessageCount < Short.MaxValue / 2) + assert(maxPendingMessageCount < Short.MaxValue / 2) - val LOG: Logger = LogUtil.getLogger(getClass, app = appId, executor = executorId, task = taskId) + private val LOG: Logger = LogUtil.getLogger(getClass, app = appId, + executor = executorId, task = taskId) import subscriber.{parallelism, partitionerDescription, processorId} @@ -62,8 +65,8 @@ class Subscription( private var life = subscriber.lifeTime - val partitioner = partitionerDescription.partitionerFactory.partitioner - val sendFn = partitioner match { + private val partitioner = partitionerDescription.partitionerFactory.partitioner + private val sendFn = partitioner match { case up: UnicastPartitioner => (msg: Message) => { val partition = up.getPartition(msg, parallelism, taskId.index) @@ -74,14 +77,13 @@ class Subscription( val partitions = mp.getPartitions(msg, parallelism, taskId.index) partitions.map(partition => sendMessage(msg, partition)).sum } - } def changeLife(life: LifeTime): Unit = { this.life = life } - def start: Unit = { + def start(): Unit = { val ackRequest = InitialAckRequest(taskId, sessionId) transport.transport(ackRequest, allTasks: _*) } @@ -91,14 +93,16 @@ class Subscription( } /** - * Return how many message is actually sent by this subscription + * Returns how many message is actually sent by this subscription + * * @param msg the message to send * @param partition the target partition to send message to * @return 1 if success */ def sendMessage(msg: Message, partition: Int): Int = { - // only send message whose timestamp matches the lifeTime + var count = 0 + // Only sends message whose timestamp matches the lifeTime if (partition != Partitioner.UNKNOWN_PARTITION_ID && life.contains(msg.timestamp)) { val targetTask = TaskId(processorId, partition) @@ -117,24 +121,25 @@ class Subscription( (messageCount(partition) + ackOnceEveryMessageCount) / maxPendingMessageCount) { sendLatencyProbe(partition) } - - return 1 + count = 1 + count } else { if (needFlush) { - flush + flush() } - - return 0 + count = 0 + count } } private var lastFlushTime: Long = 0L private val FLUSH_INTERVAL = 5 * 1000 // ms private def needFlush: Boolean = { - System.currentTimeMillis() - lastFlushTime > FLUSH_INTERVAL && Shorts.max(pendingMessageCount: _*) > 0 + System.currentTimeMillis() - lastFlushTime > FLUSH_INTERVAL && + Shorts.max(pendingMessageCount: _*) > 0 } - private def flush: Unit = { + private def flush(): Unit = { lastFlushTime = System.currentTimeMillis() allTasks.foreach { targetTaskId => sendAckRequest(targetTaskId.index) @@ -142,13 +147,14 @@ class Subscription( } private def allTasks: scala.collection.Seq[TaskId] = { - (0 until parallelism).map {taskIndex => + (0 until parallelism).map { taskIndex => TaskId(processorId, taskIndex) } } - /** Handle acknowledge message. - * Throw MessageLossException if required. + /** + * Handles acknowledge message. Throw MessageLossException if required. + * * @param ack acknowledge message received */ def receiveAck(ack: Ack): Unit = { @@ -159,7 +165,7 @@ class Subscription( if (ack.actualReceivedNum == ack.seq) { if ((ack.seq - candidateMinClockSince(index)).toShort >= 0) { if (ack.seq == messageCount(index)) { - // all messages have been acked. + // All messages have been acked. minClockValue(index) = Long.MaxValue } else { minClockValue(index) = candidateMinClock(index) @@ -171,7 +177,8 @@ class Subscription( pendingMessageCount(ack.taskId.index) = (messageCount(ack.taskId.index) - ack.seq).toShort updateMaxPendingCount() } else { - LOG.error(s"Failed! received ack: $ack, received: ${ack.actualReceivedNum}, sent: ${ack.seq}, try to replay...") + LOG.error(s"Failed! received ack: $ack, received: ${ack.actualReceivedNum}, " + + s"sent: ${ack.seq}, try to replay...") throw new MsgLostException } } @@ -181,13 +188,14 @@ class Subscription( minClockValue.min } - def allowSendingMoreMessages() : Boolean = { + def allowSendingMoreMessages(): Boolean = { maxPendingCount < maxPendingMessageCount } def sendAckRequestOnStallingTime(stallingTime: TimeStamp): Unit = { minClockValue.indices.foreach { i => - if (minClockValue(i) == stallingTime && pendingMessageCount(i) > 0 && allowSendingMoreMessages) { + if (minClockValue(i) == stallingTime && pendingMessageCount(i) > 0 + && allowSendingMoreMessages) { sendAckRequest(i) sendLatencyProbe(i) } @@ -195,7 +203,7 @@ class Subscription( } private def sendAckRequest(partition: Int): Unit = { - // we increment more count for each AckRequest + // Increments more count for each AckRequest // to throttle the number of unacked AckRequest incrementMessageCount(partition, ackOnceEveryMessageCount) val targetTask = TaskId(processorId, partition) @@ -218,11 +226,10 @@ class Subscription( val targetTask = TaskId(processorId, partition) transport.transport(probeLatency, targetTask) } - } object Subscription { - //make sure it is smaller than MAX_PENDING_MESSAGE_COUNT + // Makes sure it is smaller than MAX_PENDING_MESSAGE_COUNT final val ONE_ACKREQUEST_EVERY_MESSAGE_COUNT = 100 final val MAX_PENDING_MESSAGE_COUNT = 1000 } \ No newline at end of file diff --git a/streaming/src/main/scala/io/gearpump/streaming/task/Task.scala b/streaming/src/main/scala/io/gearpump/streaming/task/Task.scala index a176ca71d..212a6590a 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/task/Task.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/task/Task.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,14 +18,15 @@ package io.gearpump.streaming.task +import scala.concurrent.duration.FiniteDuration + import akka.actor.Actor.Receive import akka.actor.{ActorRef, ActorSystem, Cancellable, Props} +import org.slf4j.Logger + import io.gearpump.cluster.UserConfig import io.gearpump.util.LogUtil import io.gearpump.{Message, TimeStamp} -import org.slf4j.Logger - -import scala.concurrent.duration.FiniteDuration /** * This provides context information for a task. @@ -36,7 +37,7 @@ trait TaskContext { def executorId: Int - def appId : Int + def appId: Int def appName: String @@ -44,7 +45,7 @@ trait TaskContext { * The actorRef of AppMaster * @return application master's actor reference */ - def appMaster : ActorRef + def appMaster: ActorRef /** * The task parallelism @@ -61,64 +62,62 @@ trait TaskContext { */ def parallelism: Int - /** * Please don't use this if possible. * @return self actor ref */ - //TODO: We should remove the self from TaskContext + // TODO: We should remove the self from TaskContext def self: ActorRef /** * Please don't use this if possible * @return the actor system */ - //TODO: we should remove this in future + // TODO: we should remove this in future def system: ActorSystem /** - * This can be used to output messages to downstream tasks. - * The data shuffling rule can be decided by Partitioner. + * This can be used to output messages to downstream tasks. The data shuffling rule + * can be decided by Partitioner. + * * @param msg message to output */ - def output(msg : Message) : Unit - + def output(msg: Message): Unit def actorOf(props: Props): ActorRef def actorOf(props: Props, name: String): ActorRef - def schedule(initialDelay: FiniteDuration, interval: FiniteDuration)(f: ⇒ Unit): Cancellable + def schedule(initialDelay: FiniteDuration, interval: FiniteDuration)(f: => Unit): Cancellable /** * akka.actor.ActorRefProvider.scheduleOnce + * * @param initialDelay the initial delay * @param f the function to execute after initial delay * @return the executable */ - def scheduleOnce(initialDelay: FiniteDuration)(f: ⇒ Unit): Cancellable + def scheduleOnce(initialDelay: FiniteDuration)(f: => Unit): Cancellable - /** + /** * For managed message(type of Message), the sender only serve as a unique Id, * It's address is not something meaningful, you should not use this directly * * For unmanaged message, the sender represent the sender ActorRef * @return sender */ - def sender: ActorRef - + def sender: ActorRef /** - * retrieve upstream min clock from TaskActor + * Retrieves upstream min clock from TaskActor + * * @return the min clock */ def upstreamMinClock: TimeStamp - /** - * logger is environment dependant, it should be provided by + * Logger is environment dependant, it should be provided by * containing environment. - * @return */ def logger: Logger } @@ -130,28 +129,35 @@ trait TaskInterface { /** * Method called with the task is initialized. - * @param startTime startTime that can be used to decide from when a source producer task should replay the data source, or from when a processor task should recover its checkpoint data in to in-memory state. + * @param startTime startTime that can be used to decide from when a source producer task should + * replay the data source, or from when a processor task should recover its + * checkpoint data in to in-memory state. */ - def onStart(startTime : StartTime) : Unit + def onStart(startTime: StartTime): Unit - /** Method called for each message received. - * @param msg message send by upstream tasks + /** + * Method called for each message received. + * + * @param msg Message send by upstream tasks */ - def onNext(msg : Message) : Unit + def onNext(msg: Message): Unit - /** Method called when task is under clean up. + /** + * Method called when task is under clean up. + * * This can be used to cleanup resource when the application finished. */ - def onStop() : Unit + def onStop(): Unit /** - * handler for unmanaged message - * @return the handler - */ + * Handlers unmanaged messages + * + * @return the handler + */ def receiveUnManagedMessage: Receive = null } -abstract class Task(taskContext : TaskContext, userConf : UserConfig) extends TaskInterface{ +abstract class Task(taskContext: TaskContext, userConf: UserConfig) extends TaskInterface { import taskContext.{appId, executorId, taskId} @@ -170,11 +176,11 @@ abstract class Task(taskContext : TaskContext, userConf : UserConfig) extends Ta */ protected def sender: ActorRef = taskContext.sender - def onStart(startTime : StartTime) : Unit = {} + def onStart(startTime: StartTime): Unit = {} - def onNext(msg : Message) : Unit = {} + def onNext(msg: Message): Unit = {} - def onStop() : Unit = {} + def onStop(): Unit = {} override def receiveUnManagedMessage: Receive = { case msg => diff --git a/streaming/src/main/scala/io/gearpump/streaming/task/TaskActor.scala b/streaming/src/main/scala/io/gearpump/streaming/task/TaskActor.scala index d54fce57e..b9b88290b 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/task/TaskActor.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/task/TaskActor.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -22,6 +22,8 @@ import java.util import java.util.concurrent.TimeUnit import akka.actor._ +import org.slf4j.Logger + import io.gearpump.cluster.UserConfig import io.gearpump.gs.collections.impl.map.mutable.primitive.IntShortHashMap import io.gearpump.metrics.Metrics @@ -31,107 +33,107 @@ import io.gearpump.streaming.ExecutorToAppMaster._ import io.gearpump.streaming.{Constants, ProcessorId} import io.gearpump.util.{LogUtil, TimeOutScheduler} import io.gearpump.{Message, TimeStamp} -import org.slf4j.Logger /** * - * All tasks of Gearpump runs inside a Actor. - * TaskActor is the Actor container for a task. + * All tasks of Gearpump runs inside a Actor. TaskActor is the Actor container for a task. */ class TaskActor( val taskId: TaskId, - val taskContextData : TaskContextData, - userConf : UserConfig, + val taskContextData: TaskContextData, + userConf: UserConfig, val task: TaskWrapper, inputSerializerPool: SerializationFramework) - extends Actor with ExpressTransport with TimeOutScheduler{ + extends Actor with ExpressTransport with TimeOutScheduler { var upstreamMinClock: TimeStamp = 0L private var _minClock: TimeStamp = 0L def serializerPool: SerializationFramework = inputSerializerPool - import Constants._ - import io.gearpump.streaming.task.TaskActor._ import taskContextData._ + + import io.gearpump.streaming.Constants._ + import io.gearpump.streaming.task.TaskActor._ val config = context.system.settings.config val LOG: Logger = LogUtil.getLogger(getClass, app = appId, executor = executorId, task = taskId) - //metrics + // Metrics private val metricName = s"app${appId}.processor${taskId.processorId}.task${taskId.index}" - private val receiveLatency = Metrics(context.system).histogram(s"$metricName:receiveLatency", sampleRate = 1) + private val receiveLatency = Metrics(context.system).histogram( + s"$metricName:receiveLatency", sampleRate = 1) private val processTime = Metrics(context.system).histogram(s"$metricName:processTime") private val sendThroughput = Metrics(context.system).meter(s"$metricName:sendThroughput") private val receiveThroughput = Metrics(context.system).meter(s"$metricName:receiveThroughput") private val maxPendingMessageCount = config.getInt(GEARPUMP_STREAMING_MAX_PENDING_MESSAGE_COUNT) - private val ackOnceEveryMessageCount = config.getInt(GEARPUMP_STREAMING_ACK_ONCE_EVERY_MESSAGE_COUNT) + private val ackOnceEveryMessageCount = config.getInt( + GEARPUMP_STREAMING_ACK_ONCE_EVERY_MESSAGE_COUNT) private val executor = context.parent private var life = taskContextData.life - //latency probe - import context.dispatcher - + // Latency probe import scala.concurrent.duration._ + + import context.dispatcher final val LATENCY_PROBE_INTERVAL = FiniteDuration(1, TimeUnit.SECONDS) - // clock report interval + // Clock report interval final val CLOCK_REPORT_INTERVAL = FiniteDuration(1, TimeUnit.SECONDS) - // flush interval + // Flush interval final val FLUSH_INTERVAL = FiniteDuration(100, TimeUnit.MILLISECONDS) private val queue = new util.LinkedList[AnyRef]() private var subscriptions = List.empty[(Int, Subscription)] - // securityChecker will be responsible of dropping messages from + // SecurityChecker will be responsible of dropping messages from // unknown sources - private val securityChecker = new SecurityChecker(taskId, self) + private val securityChecker = new SecurityChecker(taskId, self) private[task] var sessionId = NONE_SESSION - //report to appMaster with my address + // Reports to appMaster with my address express.registerLocalActor(TaskId.toLong(taskId), self) - final def receive : Receive = null + final def receive: Receive = null task.setTaskActor(this) - def onStart(startTime : StartTime) : Unit = { + def onStart(startTime: StartTime): Unit = { task.onStart(startTime) } - def onNext(msg : Message) : Unit = task.onNext(msg) + def onNext(msg: Message): Unit = task.onNext(msg) def onUnManagedMessage(msg: Any): Unit = task.receiveUnManagedMessage.apply(msg) - def onStop() : Unit = task.onStop() + def onStop(): Unit = task.onStop() /** * output to a downstream by specifying a arrayIndex - * @param arrayIndex, this is not same as ProcessorId - * @param msg + * @param arrayIndex this is not same as ProcessorId */ - def output(arrayIndex: Int, msg: Message) : Unit = { + def output(arrayIndex: Int, msg: Message): Unit = { var count = 0 - count += this.subscriptions(arrayIndex)._2.sendMessage(msg) + count += this.subscriptions(arrayIndex)._2.sendMessage(msg) sendThroughput.mark(count) } - def output(msg : Message) : Unit = { + def output(msg: Message): Unit = { var count = 0 - this.subscriptions.foreach{ subscription => + this.subscriptions.foreach { subscription => count += subscription._2.sendMessage(msg) } sendThroughput.mark(count) } - final override def postStop() : Unit = { + final override def postStop(): Unit = { onStop() } - final override def preStart() : Unit = { + final override def preStart(): Unit = { val register = RegisterTask(taskId, executorId, local) LOG.info(s"$register") executor ! register @@ -154,7 +156,7 @@ class TaskActor( msg match { case SendAck(ack, targetTask) => transport(ack, targetTask) - case m : Message => + case m: Message => count += 1 onNext(m) case other => @@ -172,18 +174,18 @@ class TaskActor( } } - private def onStartClock: Unit = { + private def onStartClock(): Unit = { LOG.info(s"received start, clock: $upstreamMinClock, sessionId: $sessionId") subscriptions = subscribers.map { subscriber => - (subscriber.processorId , + (subscriber.processorId, new Subscription(appId, executorId, taskId, subscriber, sessionId, this, maxPendingMessageCount, ackOnceEveryMessageCount)) }.sortBy(_._1) - subscriptions.foreach(_._2.start) + subscriptions.foreach(_._2.start()) import scala.collection.JavaConverters._ - stashQueue.asScala.foreach{item => + stashQueue.asScala.foreach { item => handleMessages(item.sender).apply(item.msg) } stashQueue.clear() @@ -198,7 +200,7 @@ class TaskActor( } def waitForTaskRegistered: Receive = { - case start@ TaskRegistered(_, sessionId, startClock) => + case start@TaskRegistered(_, sessionId, startClock) => this.sessionId = sessionId this.upstreamMinClock = startClock context.become(waitForStartClock) @@ -206,9 +208,9 @@ class TaskActor( private val stashQueue = new util.LinkedList[MessageAndSender]() - def waitForStartClock : Receive = { + def waitForStartClock: Receive = { case start: StartTask => - onStartClock + onStartClock() case other: AnyRef => stashQueue.add(MessageAndSender(other, sender())) } @@ -221,8 +223,9 @@ class TaskActor( doHandleMessage() } case ackRequest: AckRequest => - //enqueue to handle the ackRequest and send back ack later - val ackResponse = securityChecker.generateAckResponse(ackRequest, sender, ackOnceEveryMessageCount) + // Enqueue to handle the ackRequest and send back ack later + val ackResponse = securityChecker.generateAckResponse(ackRequest, sender, + ackOnceEveryMessageCount) if (null != ackResponse) { queue.add(SendAck(ackResponse, ackRequest.taskId)) doHandleMessage() @@ -231,16 +234,17 @@ class TaskActor( subscriptions.find(_._1 == ack.taskId.processorId).foreach(_._2.receiveAck(ack)) doHandleMessage() case inputMessage: SerializedMessage => - val message = Message(serializerPool.get().deserialize(inputMessage.bytes), inputMessage.timeStamp) + val message = Message(serializerPool.get().deserialize(inputMessage.bytes), + inputMessage.timeStamp) receiveMessage(message, sender) case inputMessage: Message => receiveMessage(inputMessage, sender) - case upstream@ UpstreamMinClock(upstreamClock) => + case upstream@UpstreamMinClock(upstreamClock) => this.upstreamMinClock = upstreamClock val subMinClock = subscriptions.foldLeft(Long.MaxValue) { (min, sub) => val subMin = sub._2.minClock - // a subscription is holding back the _minClock; + // A subscription is holding back the _minClock; // we send AckRequest to its tasks to push _minClock forward if (subMin == _minClock) { sub._2.sendAckRequestOnStallingTime(_minClock) @@ -255,7 +259,7 @@ class TaskActor( appMaster ! update } - // check whether current task is dead. + // Checks whether current task is dead. if (_minClock > life.death) { // There will be no more message received... val unRegister = UnRegisterTask(taskId, executorId) @@ -273,11 +277,11 @@ class TaskActor( case Some(subscription) => subscription.changeLife(subscriber.lifeTime cross this.life) case None => - val subscription = new Subscription(appId, executorId, taskId, subscriber, sessionId, this, - maxPendingMessageCount, ackOnceEveryMessageCount) - subscription.start - subscriptions :+= (subscriber.processorId, subscription) - // sort, keep the order + val subscription = new Subscription(appId, executorId, taskId, subscriber, + sessionId, this, maxPendingMessageCount, ackOnceEveryMessageCount) + subscription.start() + subscriptions :+=(subscriber.processorId, subscription) + // Sorting, keep the order subscriptions = subscriptions.sortBy(_._1) } } @@ -293,12 +297,12 @@ class TaskActor( } /** - * @return min clock of this task + * Returns min clock of this task */ def minClock: TimeStamp = _minClock /** - * @return min clock of upstream task + * Returns min clock of upstream task */ def getUpstreamMinClock: TimeStamp = upstreamMinClock @@ -309,8 +313,8 @@ class TaskActor( queue.add(msg) doHandleMessage() case None => - //Todo: Indicate the error and avoid the LOG flood - //LOG.error(s"Task $taskId drop message $msg") + // TODO: Indicate the error and avoid the LOG flood + // LOG.error(s"Task $taskId drop message $msg") } } @@ -320,28 +324,28 @@ class TaskActor( } object TaskActor { - val CLOCK_SYNC_TIMEOUT_INTERVAL = 3 * 1000 //3 seconds + // 3 seconds + val CLOCK_SYNC_TIMEOUT_INTERVAL = 3 * 1000 // If the message comes from an unknown source, securityChecker will drop it - class SecurityChecker(task_id: TaskId, self : ActorRef) { + class SecurityChecker(task_id: TaskId, self: ActorRef) { private val LOG: Logger = LogUtil.getLogger(getClass, task = task_id) - // Use mutable HashMap for performance optimization + // Uses mutable HashMap for performance optimization private val receivedMsgCount = new IntShortHashMap() - // Tricky performance optimization to save memory. // We store the session Id in the uid of ActorPath // ActorPath.hashCode is same as uid. private def getSessionId(actor: ActorRef): Int = { - //TODO: As method uid is protected in [akka] package. We + // TODO: As method uid is protected in [akka] package. We // are using hashCode instead of uid. actor.hashCode() } def handleInitialAckRequest(ackRequest: InitialAckRequest): Ack = { - LOG.debug(s"Handle InitialAckRequest for session $ackRequest" ) + LOG.debug(s"Handle InitialAckRequest for session $ackRequest") val sessionId = ackRequest.sessionId if (sessionId == NONE_SESSION) { LOG.error(s"SessionId is not initialized, ackRequest: $ackRequest") @@ -355,7 +359,7 @@ object TaskActor { def generateAckResponse(ackRequest: AckRequest, sender: ActorRef, incrementCount: Int): Ack = { val sessionId = ackRequest.sessionId if (receivedMsgCount.containsKey(sessionId)) { - // we increment more count for each AckRequest + // Increments more count for each AckRequest // to throttle the number of unacked AckRequest receivedMsgCount.put(sessionId, (receivedMsgCount.get(sessionId) + incrementCount).toShort) Ack(task_id, ackRequest.seq, receivedMsgCount.get(sessionId), ackRequest.sessionId) @@ -366,8 +370,8 @@ object TaskActor { } // If the message comes from an unknown source, then drop it - def checkMessage(message : Message, sender: ActorRef): Option[Message] = { - if(sender.equals(self)){ + def checkMessage(message: Message, sender: ActorRef): Option[Message] = { + if (sender.equals(self)) { Some(message) } else { val sessionId = getSessionId(sender) diff --git a/streaming/src/main/scala/io/gearpump/streaming/task/TaskContextData.scala b/streaming/src/main/scala/io/gearpump/streaming/task/TaskContextData.scala index ce326064f..28605cf02 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/task/TaskContextData.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/task/TaskContextData.scala @@ -19,13 +19,14 @@ package io.gearpump.streaming.task import akka.actor.ActorRef + import io.gearpump.streaming.LifeTime case class TaskContextData( - executorId : Int, - appId : Int, + executorId: Int, + appId: Int, appName: String, - appMaster : ActorRef, + appMaster: ActorRef, parallelism: Int, life: LifeTime, subscribers: List[Subscriber]) diff --git a/streaming/src/main/scala/io/gearpump/streaming/task/TaskControlMessage.scala b/streaming/src/main/scala/io/gearpump/streaming/task/TaskControlMessage.scala index 59babff27..d89387a19 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/task/TaskControlMessage.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/task/TaskControlMessage.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -32,9 +32,11 @@ case class InitialAckRequest(taskId: TaskId, sessionId: Int) */ case class AckRequest(taskId: TaskId, seq: Short, sessionId: Int) -/* - Here the seq field represents the expected number of received messages - and the actualReceivedNum field means the actual received number since start +/** + * Ack back to sender task actor. + * + * @param seq The seq field represents the expected number of received messages and the + * actualReceivedNum field means the actual received number since start. */ case class Ack(taskId: TaskId, seq: Short, actualReceivedNum: Short, sessionId: Int) @@ -60,6 +62,7 @@ case object GetStartClock case class StartClock(clock: TimeStamp) +/** Probe the latency between two upstream to downstream tasks. */ case class LatencyProbe(timestamp: Long) case class SendMessageLoss() diff --git a/streaming/src/main/scala/io/gearpump/streaming/task/TaskId.scala b/streaming/src/main/scala/io/gearpump/streaming/task/TaskId.scala index f599540da..66c3c52ef 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/task/TaskId.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/task/TaskId.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,9 +20,9 @@ package io.gearpump.streaming.task import io.gearpump.streaming._ -case class TaskId(processorId : ProcessorId, index : TaskIndex) +case class TaskId(processorId: ProcessorId, index: TaskIndex) object TaskId { - def toLong(id : TaskId) = (id.processorId.toLong << 32) + id.index - def fromLong(id : Long) = TaskId(((id >> 32) & 0xFFFFFFFF).toInt, (id & 0xFFFFFFFF).toInt) + def toLong(id: TaskId): Long = (id.processorId.toLong << 32) + id.index + def fromLong(id: Long): TaskId = TaskId(((id >> 32) & 0xFFFFFFFF).toInt, (id & 0xFFFFFFFF).toInt) } \ No newline at end of file diff --git a/streaming/src/main/scala/io/gearpump/streaming/task/TaskMessageSerializer.scala b/streaming/src/main/scala/io/gearpump/streaming/task/TaskMessageSerializer.scala index 370eef742..500f8b333 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/task/TaskMessageSerializer.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/task/TaskMessageSerializer.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,8 +19,6 @@ package io.gearpump.streaming.task import java.io.{DataInput, DataOutput} -import org.jboss.netty.buffer.ChannelBuffer - trait TaskMessageSerializer[T] { def write(dataOutput: DataOutput, obj: T) diff --git a/streaming/src/main/scala/io/gearpump/streaming/task/TaskUtil.scala b/streaming/src/main/scala/io/gearpump/streaming/task/TaskUtil.scala index c6564ff94..040fc2e2f 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/task/TaskUtil.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/task/TaskUtil.scala @@ -21,11 +21,12 @@ package io.gearpump.streaming.task object TaskUtil { /** - * Resolve a classname to a Task class. + * Resolves a classname to a Task class. + * * @param className the class name to resolve * @return resolved class */ - def loadClass(className: String): Class[_<:Task] = { + def loadClass(className: String): Class[_ <: Task] = { val loader = Thread.currentThread().getContextClassLoader() loader.loadClass(className).asSubclass(classOf[Task]) } diff --git a/streaming/src/main/scala/io/gearpump/streaming/task/TaskWrapper.scala b/streaming/src/main/scala/io/gearpump/streaming/task/TaskWrapper.scala index 3be396f92..e7e883c29 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/task/TaskWrapper.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/task/TaskWrapper.scala @@ -18,22 +18,26 @@ package io.gearpump.streaming.task +import scala.concurrent.duration.FiniteDuration + import akka.actor.Actor._ import akka.actor.{ActorRef, ActorSystem, Cancellable, Props} -import io.gearpump.{TimeStamp, Message} -import io.gearpump.cluster.UserConfig -import io.gearpump.util.LogUtil import org.slf4j.Logger -import scala.concurrent.duration.FiniteDuration +import io.gearpump.cluster.UserConfig +import io.gearpump.util.LogUtil +import io.gearpump.{Message, TimeStamp} /** * This provides TaskContext for user defined tasks + * * @param taskClass task class * @param context context class * @param userConf user config */ -class TaskWrapper(val taskId: TaskId, taskClass: Class[_ <: Task], context: TaskContextData, userConf: UserConfig) extends TaskContext with TaskInterface { +class TaskWrapper( + val taskId: TaskId, taskClass: Class[_ <: Task], context: TaskContextData, + userConf: UserConfig) extends TaskContext with TaskInterface { private val LOG = LogUtil.getLogger(taskClass, task = taskId) @@ -56,18 +60,19 @@ class TaskWrapper(val taskId: TaskId, taskClass: Class[_ <: Task], context: Task override def output(msg: Message): Unit = actor.output(msg) /** - * @see [[TaskActor.output]] + * See [[io.gearpump.streaming.task.TaskActor]] output(arrayIndex: Int, msg: Message): Unit + * * @param index, not same as ProcessorId - * @param msg */ def output(index: Int, msg: Message): Unit = actor.output(index, msg) /** * Use with caution, output unmanaged message to target tasks + * * @param msg message to output * @param tasks the tasks to output to */ - def outputUnManaged(msg: AnyRef, tasks: TaskId *): Unit = { + def outputUnManaged(msg: AnyRef, tasks: TaskId*): Unit = { actor.transport(msg, tasks: _*) } @@ -105,12 +110,12 @@ class TaskWrapper(val taskId: TaskId, taskClass: Class[_ <: Task], context: Task actor.getUpstreamMinClock } - def schedule(initialDelay: FiniteDuration, interval: FiniteDuration)(f: ⇒ Unit): Cancellable = { + def schedule(initialDelay: FiniteDuration, interval: FiniteDuration)(f: => Unit): Cancellable = { val dispatcher = actor.context.system.dispatcher actor.context.system.scheduler.schedule(initialDelay, interval)(f)(dispatcher) } - def scheduleOnce(initialDelay: FiniteDuration)(f: ⇒ Unit): Cancellable = { + def scheduleOnce(initialDelay: FiniteDuration)(f: => Unit): Cancellable = { val dispatcher = actor.context.system.dispatcher actor.context.system.scheduler.scheduleOnce(initialDelay)(f)(dispatcher) } @@ -121,9 +126,8 @@ class TaskWrapper(val taskId: TaskId, taskClass: Class[_ <: Task], context: Task } /** - * logger is environment dependant, it should be provided by + * Logger is environment dependant, it should be provided by * containing environment. - * @return */ override def logger: Logger = LOG } \ No newline at end of file diff --git a/streaming/src/main/scala/io/gearpump/streaming/transaction/api/CheckpointStore.scala b/streaming/src/main/scala/io/gearpump/streaming/transaction/api/CheckpointStore.scala index 7dd08fb5d..f3894eada 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/transaction/api/CheckpointStore.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/transaction/api/CheckpointStore.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,9 +18,9 @@ package io.gearpump.streaming.transaction.api -import io.gearpump.streaming.task.TaskContext import io.gearpump.TimeStamp import io.gearpump.cluster.UserConfig +import io.gearpump.streaming.task.TaskContext /** * CheckpointStore persistently stores mapping of timestamp to checkpoint diff --git a/streaming/src/main/scala/io/gearpump/streaming/transaction/api/MessageDecoder.scala b/streaming/src/main/scala/io/gearpump/streaming/transaction/api/MessageDecoder.scala index 1869eab34..7039b71e5 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/transaction/api/MessageDecoder.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/transaction/api/MessageDecoder.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,8 +21,7 @@ package io.gearpump.streaming.transaction.api import io.gearpump.Message /** - * MessageDecoder decodes raw bytes to Message - * It is usually written by end user and + * MessageDecoder decodes raw bytes to Message It is usually written by end user and * passed into TimeReplayableSource */ trait MessageDecoder extends java.io.Serializable { diff --git a/streaming/src/main/scala/io/gearpump/streaming/transaction/api/OffsetManager.scala b/streaming/src/main/scala/io/gearpump/streaming/transaction/api/OffsetManager.scala index 29656dd7a..412ddcc31 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/transaction/api/OffsetManager.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/transaction/api/OffsetManager.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,28 +18,27 @@ package io.gearpump.streaming.transaction.api -import io.gearpump.{Message, TimeStamp} - import scala.util.Try +import io.gearpump.{Message, TimeStamp} + /** - * filter offsets and store the mapping from timestamp to offset + * Filters offsets and store the mapping from timestamp to offset */ trait MessageFilter { def filter(messageAndOffset: (Message, Long)): Option[Message] } /** - * resolve timestamp to offset by look up the underlying storage + * Resolves timestamp to offset by look up the underlying storage */ trait OffsetTimeStampResolver { def resolveOffset(time: TimeStamp): Try[Long] } /** - * manages message's offset on TimeReplayableSource and timestamp + * Manages message's offset on TimeReplayableSource and timestamp */ trait OffsetManager extends MessageFilter with OffsetTimeStampResolver { def close(): Unit -} - +} \ No newline at end of file diff --git a/streaming/src/main/scala/io/gearpump/streaming/transaction/api/OffsetStorage.scala b/streaming/src/main/scala/io/gearpump/streaming/transaction/api/OffsetStorage.scala index aa9966eec..fa7161c0f 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/transaction/api/OffsetStorage.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/transaction/api/OffsetStorage.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,10 +18,10 @@ package io.gearpump.streaming.transaction.api -import io.gearpump.TimeStamp - import scala.util.Try +import io.gearpump.TimeStamp + object OffsetStorage { /** @@ -47,15 +47,17 @@ object OffsetStorage { */ trait OffsetStorage { /** - * try to look up the time in the OffsetStorage - * return the corresponding Offset if the time is - * in the range of stored TimeStamps or one of the - * failure info (StorageEmpty, Overflow, Underflow) + * Tries to look up the time in the OffsetStorage return the corresponding Offset if the time is + * in the range of stored TimeStamps or one of the failure info (StorageEmpty, Overflow, + * Underflow) + * * @param time the time to look for * @return the corresponding offset if the time is in the range, otherwise failure */ def lookUp(time: TimeStamp): Try[Array[Byte]] + def append(time: TimeStamp, offset: Array[Byte]): Unit + def close(): Unit } diff --git a/streaming/src/main/scala/io/gearpump/streaming/transaction/api/TimeReplayableSource.scala b/streaming/src/main/scala/io/gearpump/streaming/transaction/api/TimeReplayableSource.scala index b6c1e6c5c..50711ee17 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/transaction/api/TimeReplayableSource.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/transaction/api/TimeReplayableSource.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,11 +21,10 @@ package io.gearpump.streaming.transaction.api import io.gearpump.streaming.source.DataSource /** - * AT-LEAST-ONCE API + * AT-LEAST-ONCE API. Represents a data source which allow replaying. * - * subclass should be able to replay messages on recovery from the time - * when an application crashed + * Subclass should be able to replay messages on recovery from the time + * when an application crashed. */ trait TimeReplayableSource extends DataSource - diff --git a/streaming/src/main/scala/io/gearpump/streaming/transaction/api/TimeStampFilter.scala b/streaming/src/main/scala/io/gearpump/streaming/transaction/api/TimeStampFilter.scala index 7df508cc2..7c34e1afc 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/transaction/api/TimeStampFilter.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/transaction/api/TimeStampFilter.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -21,10 +21,9 @@ package io.gearpump.streaming.transaction.api import io.gearpump.{Message, TimeStamp} /** - * TimeStampFilter filters message comparing its TimeStamp with the predicate. + * TimeStampFilter filters out messages that are obsolete. */ trait TimeStampFilter extends java.io.Serializable { def filter(msg: Message, predicate: TimeStamp): Option[Message] } - diff --git a/streaming/src/main/scala/io/gearpump/streaming/util/ActorPathUtil.scala b/streaming/src/main/scala/io/gearpump/streaming/util/ActorPathUtil.scala index 5edcf3861..c2ac32a29 100644 --- a/streaming/src/main/scala/io/gearpump/streaming/util/ActorPathUtil.scala +++ b/streaming/src/main/scala/io/gearpump/streaming/util/ActorPathUtil.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,16 +19,22 @@ package io.gearpump.streaming.util import akka.actor.{ActorPath, ActorRef} + import io.gearpump.streaming.task.TaskId object ActorPathUtil { - def executorActorName(executorId: Int) = executorId.toString + def executorActorName(executorId: Int): String = executorId.toString - def taskActorName(taskId: TaskId) = s"processor_${taskId.processorId}_task_${taskId.index}" + def taskActorName(taskId: TaskId): String = { + s"processor_${taskId.processorId}_task_${taskId.index}" + } def taskActorPath(appMaster: ActorRef, executorId: Int, taskId: TaskId): ActorPath = { - appMaster.path.child(executorManagerActorName).child(executorActorName(executorId)).child(taskActorName(taskId)) + val executorManager = appMaster.path.child(executorManagerActorName) + val executor = executorManager.child(executorActorName(executorId)) + val task = executor.child(taskActorName(taskId)) + task } def executorManagerActorName: String = "executors" diff --git a/streaming/src/test/scala/io/gearpump/streaming/DAGSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/DAGSpec.scala index 0207fb399..13b8e3497 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/DAGSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/DAGSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,13 +18,14 @@ package io.gearpump.streaming +import org.scalacheck.Gen +import org.scalatest.prop.PropertyChecks +import org.scalatest.{Matchers, PropSpec} + import io.gearpump.partitioner.PartitionerDescription import io.gearpump.streaming.task.TaskId import io.gearpump.util.Graph import io.gearpump.util.Graph.Node -import org.scalacheck.Gen -import org.scalatest.prop.PropertyChecks -import org.scalatest.{Matchers, PropSpec} class DAGSpec extends PropSpec with PropertyChecks with Matchers { diff --git a/streaming/src/test/scala/io/gearpump/streaming/MessageSerializerSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/MessageSerializerSpec.scala index a5067f282..7938415b3 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/MessageSerializerSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/MessageSerializerSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,11 +17,12 @@ */ package io.gearpump.streaming -import io.gearpump.streaming.task._ -import io.gearpump.transport.netty.WrappedChannelBuffer import org.jboss.netty.buffer.{ChannelBufferOutputStream, ChannelBuffers} import org.scalatest.{Matchers, WordSpec} +import io.gearpump.streaming.task._ +import io.gearpump.transport.netty.WrappedChannelBuffer + class MessageSerializerSpec extends WordSpec with Matchers { def testSerializer[T](obj: T, taskMessageSerializer: TaskMessageSerializer[T]): T = { @@ -32,7 +33,7 @@ class MessageSerializerSpec extends WordSpec with Matchers { taskMessageSerializer.read(input) } - "SerializedMessageSerializer" should { + "SerializedMessageSerializer" should { "serialize and deserialize SerializedMessage properly" in { val serializer = new SerializedMessageSerializer val data = new Array[Byte](256) @@ -43,7 +44,7 @@ class MessageSerializerSpec extends WordSpec with Matchers { } } - "TaskIdSerializer" should { + "TaskIdSerializer" should { "serialize and deserialize TaskId properly" in { val taskIdSerializer = new TaskIdSerializer val taskId = TaskId(1, 3) @@ -51,7 +52,7 @@ class MessageSerializerSpec extends WordSpec with Matchers { } } - "AckRequestSerializer" should { + "AckRequestSerializer" should { "serialize and deserialize AckRequest properly" in { val serializer = new AckRequestSerializer val ackRequest = AckRequest(TaskId(1, 2), 1000, 1024) @@ -59,7 +60,7 @@ class MessageSerializerSpec extends WordSpec with Matchers { } } - "InitialAckRequestSerializer" should { + "InitialAckRequestSerializer" should { "serialize and deserialize AckRequest properly" in { val serializer = new InitialAckRequestSerializer val ackRequest = InitialAckRequest(TaskId(1, 2), 1024) @@ -67,7 +68,7 @@ class MessageSerializerSpec extends WordSpec with Matchers { } } - "AckSerializer" should { + "AckSerializer" should { "serialize and deserialize Ack properly" in { val serializer = new AckSerializer val ack = Ack(TaskId(1, 2), 1024, 1023, 1799) diff --git a/streaming/src/test/scala/io/gearpump/streaming/MockUtil.scala b/streaming/src/test/scala/io/gearpump/streaming/MockUtil.scala index 6577718dd..40310f798 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/MockUtil.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/MockUtil.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,14 +17,12 @@ */ package io.gearpump.streaming -import akka.actor.{ActorSystem, Actor} -import akka.testkit.{TestProbe, TestActorRef} -import io.gearpump.streaming.task.TaskContext -import io.gearpump.streaming.task.TaskId +import akka.actor.{Actor, ActorSystem} +import akka.testkit.TestActorRef +import org.mockito.{ArgumentMatcher, Matchers, Mockito} + import io.gearpump.cluster.TestUtil -import org.mockito.{Mockito, ArgumentMatcher} -import org.mockito.Mockito -import org.mockito.Matchers +import io.gearpump.streaming.task.{TaskContext, TaskId} object MockUtil { @@ -39,7 +37,7 @@ object MockUtil { context } - def argMatch[T](func: T => Boolean) : T = { + def argMatch[T](func: T => Boolean): T = { Matchers.argThat(new ArgumentMatcher[T] { override def matches(param: Any): Boolean = { val mesage = param.asInstanceOf[T] diff --git a/streaming/src/test/scala/io/gearpump/streaming/StreamingTestUtil.scala b/streaming/src/test/scala/io/gearpump/streaming/StreamingTestUtil.scala index 6866907a7..2ea8b84c2 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/StreamingTestUtil.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/StreamingTestUtil.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,10 +19,11 @@ package io.gearpump.streaming import akka.actor._ import akka.testkit.TestActorRef + import io.gearpump.cluster.AppMasterToMaster.RegisterAppMaster import io.gearpump.cluster.appmaster.AppMasterRuntimeInfo import io.gearpump.cluster.scheduler.Resource -import io.gearpump.cluster.{MiniCluster, AppDescription, AppMasterContext, UserConfig} +import io.gearpump.cluster.{AppDescription, AppMasterContext, MiniCluster, UserConfig} import io.gearpump.streaming.appmaster.AppMaster import io.gearpump.util.Graph @@ -33,7 +34,8 @@ object StreamingTestUtil { def startAppMaster(miniCluster: MiniCluster, appId: Int): TestActorRef[AppMaster] = { implicit val actorSystem = miniCluster.system - val masterConf = AppMasterContext(appId, testUserName, Resource(1), null, None,miniCluster.mockMaster,AppMasterRuntimeInfo(appId, appName = appId.toString)) + val masterConf = AppMasterContext(appId, testUserName, Resource(1), null, + None, miniCluster.mockMaster, AppMasterRuntimeInfo(appId, appName = appId.toString)) val app = StreamApplication("test", Graph.empty, UserConfig.empty) val appDescription = AppDescription(app.name, app.appMaster.getName, app.userConfig) diff --git a/streaming/src/test/scala/io/gearpump/streaming/appmaster/AppMasterSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/appmaster/AppMasterSpec.scala index d1d7006d1..90fb8ab26 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/appmaster/AppMasterSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/appmaster/AppMasterSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,9 +17,13 @@ */ package io.gearpump.streaming.appmaster +import scala.concurrent.duration._ + import akka.actor.{ActorRef, Props} import akka.testkit.{TestActorRef, TestProbe} -import io.gearpump.{WorkerId, Message} +import org.scalatest._ + +import io.gearpump.Message import io.gearpump.cluster.AppMasterToMaster._ import io.gearpump.cluster.AppMasterToWorker.LaunchExecutor import io.gearpump.cluster.ClientToMaster.ShutdownApplication @@ -29,19 +33,16 @@ import io.gearpump.cluster._ import io.gearpump.cluster.appmaster.{AppMasterRuntimeEnvironment, AppMasterRuntimeInfo} import io.gearpump.cluster.master.MasterProxy import io.gearpump.cluster.scheduler.{Resource, ResourceAllocation, ResourceRequest} +import io.gearpump.cluster.worker.WorkerId import io.gearpump.jarstore.FilePath import io.gearpump.partitioner.HashPartitioner import io.gearpump.streaming.task.{StartTime, TaskContext, _} import io.gearpump.streaming.{Processor, StreamApplication} import io.gearpump.util.Graph import io.gearpump.util.Graph._ -import org.scalatest._ - -import scala.concurrent.duration._ -import scala.language.postfixOps class AppMasterSpec extends WordSpec with Matchers with BeforeAndAfterEach with MasterHarness { - override def config = TestUtil.DEFAULT_CONFIG + protected override def config = TestUtil.DEFAULT_CONFIG var appMaster: ActorRef = null @@ -63,7 +64,7 @@ class AppMasterSpec extends WordSpec with Matchers with BeforeAndAfterEach with var appMasterContext: AppMasterContext = null var appMasterRuntimeInfo: AppMasterRuntimeInfo = null - override def beforeEach() = { + override def beforeEach(): Unit = { startActorSystem() mockTask = TestProbe()(getActorSystem) @@ -76,50 +77,54 @@ class AppMasterSpec extends WordSpec with Matchers with BeforeAndAfterEach with implicit val system = getActorSystem conf = UserConfig.empty.withValue(AppMasterSpec.MASTER, mockMaster.ref) val mockJar = AppJar("for_test", FilePath("path")) - appMasterContext = AppMasterContext(appId, "test", resource, null, Some(mockJar), mockMaster.ref, appMasterRuntimeInfo) + appMasterContext = AppMasterContext(appId, "test", resource, null, Some(mockJar), + mockMaster.ref, appMasterRuntimeInfo) val graph = Graph(taskDescription1 ~ partitioner ~> taskDescription2) val streamApp = StreamApplication("test", graph, conf) appDescription = Application.ApplicationToAppDescription(streamApp) import scala.concurrent.duration._ - mockMasterProxy = getActorSystem.actorOf( - Props(new MasterProxy(List(mockMaster.ref.path), 30 seconds)), AppMasterSpec.MOCK_MASTER_PROXY) + mockMasterProxy = getActorSystem.actorOf(Props(new MasterProxy(List(mockMaster.ref.path), + 30.seconds)), AppMasterSpec.MOCK_MASTER_PROXY) TestActorRef[AppMaster]( - AppMasterRuntimeEnvironment.props(List(mockMasterProxy.path), appDescription, appMasterContext))(getActorSystem) + AppMasterRuntimeEnvironment.props(List(mockMasterProxy.path), appDescription, + appMasterContext))(getActorSystem) - val registerAppMaster = mockMaster.receiveOne(15 seconds) + val registerAppMaster = mockMaster.receiveOne(15.seconds) assert(registerAppMaster.isInstanceOf[RegisterAppMaster]) appMaster = registerAppMaster.asInstanceOf[RegisterAppMaster].appMaster mockMaster.reply(AppMasterRegistered(appId)) - mockMaster.expectMsg(15 seconds, GetAppData(appId, "DAG")) + mockMaster.expectMsg(15.seconds, GetAppData(appId, "DAG")) mockMaster.reply(GetAppDataResult("DAG", null)) - mockMaster.expectMsg(15 seconds, GetAppData(appId, "startClock")) + mockMaster.expectMsg(15.seconds, GetAppData(appId, "startClock")) mockMaster.reply(GetAppDataResult("startClock", 0L)) - mockMaster.expectMsg(15 seconds, RequestResource(appId, ResourceRequest(Resource(4), workerId = WorkerId.unspecified))) + mockMaster.expectMsg(15.seconds, RequestResource(appId, ResourceRequest(Resource(4), + workerId = WorkerId.unspecified))) } - override def afterEach() = { + override def afterEach(): Unit = { shutdownActorSystem() } "AppMaster" should { "kill it self when allocate resource time out" in { - mockMaster.reply(ResourceAllocated(Array(ResourceAllocation(Resource(2), mockWorker.ref, workerId)))) - mockMaster.expectMsg(60 seconds, ShutdownApplication(appId)) + mockMaster.reply(ResourceAllocated(Array(ResourceAllocation(Resource(2), + mockWorker.ref, workerId)))) + mockMaster.expectMsg(60.seconds, ShutdownApplication(appId)) } "reschedule the resource when the worker reject to start executor" in { val resource = Resource(4) - mockMaster.reply(ResourceAllocated(Array(ResourceAllocation(resource, mockWorker.ref, workerId)))) + mockMaster.reply(ResourceAllocated(Array(ResourceAllocation(resource, + mockWorker.ref, workerId)))) mockWorker.expectMsgClass(classOf[LaunchExecutor]) mockWorker.reply(ExecutorLaunchRejected("")) mockMaster.expectMsg(RequestResource(appId, ResourceRequest(resource, WorkerId.unspecified))) } "find a new master when lost connection with master" in { - println(config.getList("akka.loggers")) val watcher = TestProbe()(getActorSystem) watcher.watch(mockMasterProxy) @@ -130,74 +135,74 @@ class AppMasterSpec extends WordSpec with Matchers with BeforeAndAfterEach with Thread.sleep(2000) import scala.concurrent.duration._ - mockMasterProxy = getActorSystem.actorOf(Props(new MasterProxy(List(mockMaster.ref.path), 30 seconds)), AppMasterSpec.MOCK_MASTER_PROXY) - mockMaster.expectMsgClass(15 seconds, classOf[RegisterAppMaster]) - } - - /* - - TODO: This test is failing on Travis randomly - We have not identifed the root cause. - Check: https://travis-ci.org/intel-hadoop/gearpump/builds/56826843 - Issue tracker: https://github.com/intel-hadoop/gearpump/issues/733 - - "launch executor and task properly" in { - mockMaster.reply(ResourceAllocated(Array(ResourceAllocation(Resource(4), mockWorker.ref, workerId)))) - mockWorker.expectMsgClass(classOf[LaunchExecutor]) - - val workerSystem = ActorSystem("worker", TestUtil.DEFAULT_CONFIG) - mockWorker.reply(RegisterActorSystem(ActorUtil.getSystemAddress(workerSystem).toString)) - for (i <- 1 to 4) { - mockMaster.expectMsg(10 seconds, AppMasterSpec.TaskStarted) - } - - //clock status: task(0,0) -> 1, task(0,1)->0, task(1, 0)->0, task(1,1)->0 - appMaster.tell(UpdateClock(TaskId(0, 0), 1), mockTask.ref) - - //there is no further upstream, so the upstreamMinClock is Long.MaxValue - mockTask.expectMsg(UpstreamMinClock(Long.MaxValue)) - - // check min clock - appMaster.tell(GetLatestMinClock, mockTask.ref) - mockTask.expectMsg(LatestMinClock(0)) - - - //clock status: task(0,0) -> 1, task(0,1)->1, task(1, 0)->0, task(1,1)->0 - appMaster.tell(UpdateClock(TaskId(0, 1), 1), mockTask.ref) - - //there is no further upstream, so the upstreamMinClock is Long.MaxValue - mockTask.expectMsg(UpstreamMinClock(Long.MaxValue)) - - // check min clock - appMaster.tell(GetLatestMinClock, mockTask.ref) - mockTask.expectMsg(LatestMinClock(0)) - - //clock status: task(0,0) -> 1, task(0,1)->1, task(1, 1)->0, task(1,1)->0 - appMaster.tell(UpdateClock(TaskId(1, 0), 1), mockTask.ref) - - // min clock of processor 0 (Task(0, 0) and Task(0, 1)) - mockTask.expectMsg(UpstreamMinClock(1)) - - // check min clock - appMaster.tell(GetLatestMinClock, mockTask.ref) - mockTask.expectMsg(LatestMinClock(0)) - - //clock status: task(0,0) -> 1, task(0,1)->1, task(1, 1)->0, task(1,1)->1 - appMaster.tell(UpdateClock(TaskId(1, 1), 1), mockTask.ref) - - // min clock of processor 0 (Task(0, 0) and Task(0, 1)) - mockTask.expectMsg(UpstreamMinClock(1)) - - // check min clock - appMaster.tell(GetLatestMinClock, mockTask.ref) - mockTask.expectMsg(LatestMinClock(1)) - - //shutdown worker and all executor on this work, expect appmaster to ask for new resources - workerSystem.shutdown() - mockMaster.expectMsg(RequestResource(appId, ResourceRequest(Resource(4), relaxation = Relaxation.ONEWORKER))) + mockMasterProxy = getActorSystem.actorOf(Props(new MasterProxy(List(mockMaster.ref.path), + 30.seconds)), AppMasterSpec.MOCK_MASTER_PROXY) + mockMaster.expectMsgClass(15.seconds, classOf[RegisterAppMaster]) } -**/ + // // TODO: This test is failing on Travis randomly + // // We have not identifed the root cause. + // // Check: https://travis-ci.org/intel-hadoop/gearpump/builds/56826843 + // // Issue tracker: https://github.com/intel-hadoop/gearpump/issues/733 + // + // "launch executor and task properly" in { + // mockMaster.reply(ResourceAllocated(Array(ResourceAllocation(Resource(4), mockWorker.ref, + // workerId)))) + // mockWorker.expectMsgClass(classOf[LaunchExecutor]) + // + // val workerSystem = ActorSystem("worker", TestUtil.DEFAULT_CONFIG) + // mockWorker.reply(RegisterActorSystem(ActorUtil.getSystemAddress(workerSystem).toString)) + // for (i <- 1 to 4) { + // mockMaster.expectMsg(10 seconds, AppMasterSpec.TaskStarted) + // } + // + // // clock status: task(0,0) -> 1, task(0,1)->0, task(1, 0)->0, task(1,1)->0 + // appMaster.tell(UpdateClock(TaskId(0, 0), 1), mockTask.ref) + // + // // there is no further upstream, so the upstreamMinClock is Long.MaxValue + // mockTask.expectMsg(UpstreamMinClock(Long.MaxValue)) + // + // // check min clock + // appMaster.tell(GetLatestMinClock, mockTask.ref) + // mockTask.expectMsg(LatestMinClock(0)) + // + // + // // clock status: task(0,0) -> 1, task(0,1)->1, task(1, 0)->0, task(1,1)->0 + // appMaster.tell(UpdateClock(TaskId(0, 1), 1), mockTask.ref) + // + // // there is no further upstream, so the upstreamMinClock is Long.MaxValue + // mockTask.expectMsg(UpstreamMinClock(Long.MaxValue)) + // + // // check min clock + // appMaster.tell(GetLatestMinClock, mockTask.ref) + // mockTask.expectMsg(LatestMinClock(0)) + // + // // Clock status: task(0,0) -> 1, task(0,1)->1, task(1, 1)->0, task(1,1)->0 + // appMaster.tell(UpdateClock(TaskId(1, 0), 1), mockTask.ref) + // + // // Min clock of processor 0 (Task(0, 0) and Task(0, 1)) + // mockTask.expectMsg(UpstreamMinClock(1)) + // + // // check min clock + // appMaster.tell(GetLatestMinClock, mockTask.ref) + // mockTask.expectMsg(LatestMinClock(0)) + // + // // clock status: task(0,0) -> 1, task(0,1)->1, task(1, 1)->0, task(1,1)->1 + // appMaster.tell(UpdateClock(TaskId(1, 1), 1), mockTask.ref) + // + // // min clock of processor 0 (Task(0, 0) and Task(0, 1)) + // mockTask.expectMsg(UpstreamMinClock(1)) + // + // // check min clock + // appMaster.tell(GetLatestMinClock, mockTask.ref) + // mockTask.expectMsg(LatestMinClock(1)) + // + // // shutdown worker and all executor on this work, expect appmaster to ask + // // for new resources + // workerSystem.shutdown() + // mockMaster.expectMsg(RequestResource(appId, ResourceRequest(Resource(4), relaxation = + // Relaxation.ONEWORKER))) + // } } def ignoreSaveAppData: PartialFunction[Any, Boolean] = { @@ -212,7 +217,7 @@ object AppMasterSpec { val MOCK_MASTER_PROXY = "mockMasterProxy" } -class TaskA(taskContext : TaskContext, userConf : UserConfig) extends Task(taskContext, userConf) { +class TaskA(taskContext: TaskContext, userConf: UserConfig) extends Task(taskContext, userConf) { val master = userConf.getValue[ActorRef](AppMasterSpec.MASTER).get override def onStart(startTime: StartTime): Unit = { @@ -222,7 +227,7 @@ class TaskA(taskContext : TaskContext, userConf : UserConfig) extends Task(taskC override def onNext(msg: Message): Unit = {} } -class TaskB(taskContext : TaskContext, userConf : UserConfig) extends Task(taskContext, userConf) { +class TaskB(taskContext: TaskContext, userConf: UserConfig) extends Task(taskContext, userConf) { val master = userConf.getValue[ActorRef](AppMasterSpec.MASTER).get override def onStart(startTime: StartTime): Unit = { diff --git a/streaming/src/test/scala/io/gearpump/streaming/appmaster/ClockServiceSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/appmaster/ClockServiceSpec.scala index a4510533b..01744a300 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/appmaster/ClockServiceSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/appmaster/ClockServiceSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,24 +17,24 @@ */ package io.gearpump.streaming.appmaster +import scala.concurrent.{Future, Promise} + import akka.actor.{ActorSystem, Props} import akka.testkit.{ImplicitSender, TestKit, TestProbe} -import io.gearpump.streaming.task.{GetStartClock, UpstreamMinClock, GetLatestMinClock} -import io.gearpump.cluster.{UserConfig, TestUtil} +import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} + +import io.gearpump.cluster.{TestUtil, UserConfig} import io.gearpump.partitioner.{HashPartitioner, Partitioner, PartitionerDescription} import io.gearpump.streaming.appmaster.ClockService.{ChangeToNewDAG, ChangeToNewDAGSuccess, HealthChecker, ProcessorClock} import io.gearpump.streaming.appmaster.ClockServiceSpec.Store import io.gearpump.streaming.storage.AppDataStore -import io.gearpump.streaming.task._ -import io.gearpump.streaming.{LifeTime, DAG, ProcessorDescription} +import io.gearpump.streaming.task.{GetLatestMinClock, GetStartClock, UpstreamMinClock, _} +import io.gearpump.streaming.{DAG, LifeTime, ProcessorDescription} import io.gearpump.util.Graph import io.gearpump.util.Graph._ -import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} - -import scala.concurrent.{Future, Promise} class ClockServiceSpec(_system: ActorSystem) extends TestKit(_system) with ImplicitSender - with WordSpecLike with Matchers with BeforeAndAfterAll{ + with WordSpecLike with Matchers with BeforeAndAfterAll { def this() = this(ActorSystem("ClockServiceSpec", TestUtil.DEFAULT_CONFIG)) @@ -50,46 +50,48 @@ class ClockServiceSpec(_system: ActorSystem) extends TestKit(_system) with Impli "The ClockService" should { "maintain a global view of message timestamp in the application" in { val store = new Store() - val startClock = 100L + val startClock = 100L store.put(ClockService.START_CLOCK, startClock) val clockService = system.actorOf(Props(new ClockService(dag, store))) clockService ! GetLatestMinClock expectMsg(LatestMinClock(startClock)) - //task(0,0): clock(101); task(1,0): clock(100) + // task(0,0): clock(101); task(1,0): clock(100) clockService ! UpdateClock(TaskId(0, 0), 101) - // there is no upstream, so pick Long.MaxValue + // There is no upstream, so pick Long.MaxValue expectMsg(UpstreamMinClock(Long.MaxValue)) - // min clock is updated + // Min clock is updated clockService ! GetLatestMinClock expectMsg(LatestMinClock(100)) - - //task(0,0): clock(101); task(1,0): clock(101) + // task(0,0): clock(101); task(1,0): clock(101) clockService ! UpdateClock(TaskId(1, 0), 101) - //upstream is Task(0, 0), 101 + // Upstream is Task(0, 0), 101 expectMsg(UpstreamMinClock(101)) - // min clock is updated + // Min clock is updated clockService ! GetLatestMinClock expectMsg(LatestMinClock(101)) } "act on ChangeToNewDAG and make sure downstream clock smaller than upstreams" in { val store = new Store() - val startClock = 100L + val startClock = 100L store.put(ClockService.START_CLOCK, startClock) val clockService = system.actorOf(Props(new ClockService(dag, store))) val task = TestProbe() clockService.tell(UpdateClock(TaskId(0, 0), 200), task.ref) task.expectMsgType[UpstreamMinClock] - val task3 = ProcessorDescription(id = 3, taskClass = classOf[TaskActor].getName, parallelism = 1) - val task4 = ProcessorDescription(id = 4, taskClass = classOf[TaskActor].getName, parallelism = 1) - val task5 = ProcessorDescription(id = 5, taskClass = classOf[TaskActor].getName, parallelism = 1) + val task3 = ProcessorDescription(id = 3, taskClass = classOf[TaskActor].getName, + parallelism = 1) + val task4 = ProcessorDescription(id = 4, taskClass = classOf[TaskActor].getName, + parallelism = 1) + val task5 = ProcessorDescription(id = 5, taskClass = classOf[TaskActor].getName, + parallelism = 1) val dagAddMiddleNode = DAG(Graph( task1 ~ hash ~> task2, task1 ~ hash ~> task3, @@ -100,12 +102,12 @@ class ClockServiceSpec(_system: ActorSystem) extends TestKit(_system) with Impli val user = TestProbe() clockService.tell(ChangeToNewDAG(dagAddMiddleNode), user.ref) - val clocks = user.expectMsgPF(){ + val clocks = user.expectMsgPF() { case ChangeToNewDAGSuccess(clocks) => clocks } - // for intermediate task, pick its upstream as initial clock + // For intermediate task, pick its upstream as initial clock assert(clocks(task3.id) == clocks(task1.id)) // For sink task, pick its upstream as initial clock @@ -117,7 +119,7 @@ class ClockServiceSpec(_system: ActorSystem) extends TestKit(_system) with Impli "maintain global checkpoint time" in { val store = new Store() - val startClock = 100L + val startClock = 100L store.put(ClockService.START_CLOCK, startClock) val clockService = system.actorOf(Props(new ClockService(dag, store))) clockService ! UpdateClock(TaskId(0, 0), 200L) @@ -129,8 +131,10 @@ class ClockServiceSpec(_system: ActorSystem) extends TestKit(_system) with Impli expectMsg(StartClock(200L)) val conf = UserConfig.empty.withBoolean("state.checkpoint.enable", true) - val task3 = ProcessorDescription(id = 3, taskClass = classOf[TaskActor].getName, parallelism = 1, taskConf = conf) - val task4 = ProcessorDescription(id = 4, taskClass = classOf[TaskActor].getName, parallelism = 1, taskConf = conf) + val task3 = ProcessorDescription(id = 3, taskClass = classOf[TaskActor].getName, + parallelism = 1, taskConf = conf) + val task4 = ProcessorDescription(id = 4, taskClass = classOf[TaskActor].getName, + parallelism = 1, taskConf = conf) val dagWithStateTasks = DAG(Graph( task1 ~ hash ~> task2, task1 ~ hash ~> task3, @@ -184,14 +188,14 @@ class ClockServiceSpec(_system: ActorSystem) extends TestKit(_system) with Impli val sourceClock = new ProcessorClock(0, LifeTime.Immortal, 1) sourceClock.init(0L) val sink = ProcessorDescription(id = 1, taskClass = null, parallelism = 1) - val sinkClock = new ProcessorClock(1,LifeTime.Immortal, 1) + val sinkClock = new ProcessorClock(1, LifeTime.Immortal, 1) sinkClock.init(0L) val graph = Graph.empty[ProcessorDescription, PartitionerDescription] graph.addVertex(source) graph.addVertex(sink) graph.addEdge(source, PartitionerDescription(null), sink) val dag = DAG(graph) - val clocks = Map ( + val clocks = Map( 0 -> sourceClock, 1 -> sinkClock ) @@ -199,30 +203,29 @@ class ClockServiceSpec(_system: ActorSystem) extends TestKit(_system) with Impli sourceClock.updateMinClock(0, 100L) sinkClock.updateMinClock(0, 100L) - // clock advance from 0 to 100, there is no stalling. + // Clock advances from 0 to 100, there is no stalling. healthChecker.check(currentMinClock = 100, clocks, dag, 200) healthChecker.getReport.stallingTasks shouldBe List.empty[TaskId] - // clock not advancing. - // pasted time exceed the stalling threshold, report stalling + // Clock not advancing. + // Pasted time exceed the stalling threshold, report stalling healthChecker.check(currentMinClock = 100, clocks, dag, 1300) - // the source task is stalling the clock + // The source task is stalling the clock healthChecker.getReport.stallingTasks shouldBe List(TaskId(0, 0)) - // advance the source clock + // Advance the source clock sourceClock.updateMinClock(0, 101L) healthChecker.check(currentMinClock = 100, clocks, dag, 1300) - // the sink task is stalling the clock + // The sink task is stalling the clock healthChecker.getReport.stallingTasks shouldBe List(TaskId(1, 0)) } } - } object ClockServiceSpec { - class Store extends AppDataStore{ + class Store extends AppDataStore { private var map = Map.empty[String, Any] @@ -231,7 +234,7 @@ object ClockServiceSpec { Promise.successful(value).future } - def get(key: String) : Future[Any] = { + def get(key: String): Future[Any] = { Promise.successful(map.get(key).getOrElse(null)).future } } diff --git a/streaming/src/test/scala/io/gearpump/streaming/appmaster/DagManagerSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/appmaster/DagManagerSpec.scala index 2750cc55b..fb633f9a7 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/appmaster/DagManagerSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/appmaster/DagManagerSpec.scala @@ -18,8 +18,13 @@ package io.gearpump.streaming.appmaster +import scala.concurrent.Await +import scala.concurrent.duration.Duration + import akka.actor.{ActorSystem, Props} import akka.testkit.TestProbe +import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} + import io.gearpump.cluster.{TestUtil, UserConfig} import io.gearpump.partitioner.{HashPartitioner, Partitioner} import io.gearpump.streaming.appmaster.DagManager.{DAGOperationFailed, DAGOperationSuccess, GetLatestDAG, GetTaskLaunchData, LatestDAG, NewDAGDeployed, ReplaceProcessor, TaskLaunchData, WatchChange} @@ -27,7 +32,6 @@ import io.gearpump.streaming.task.{Subscriber, TaskActor} import io.gearpump.streaming.{DAG, LifeTime, ProcessorDescription, StreamApplication} import io.gearpump.util.Graph import io.gearpump.util.Graph._ -import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} class DagManagerSpec extends WordSpecLike with Matchers with BeforeAndAfterAll { @@ -94,8 +98,8 @@ class DagManagerSpec extends WordSpecLike with Matchers with BeforeAndAfterAll { } override def afterAll { - system.shutdown() - system.awaitTermination() + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } override def beforeAll { diff --git a/streaming/src/test/scala/io/gearpump/streaming/appmaster/ExecutorManagerSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/appmaster/ExecutorManagerSpec.scala index 9121a22af..a57a1ae63 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/appmaster/ExecutorManagerSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/appmaster/ExecutorManagerSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,39 +18,43 @@ package io.gearpump.streaming.appmaster +import scala.concurrent.Await +import scala.concurrent.duration.Duration + import akka.actor._ import akka.testkit.TestProbe import com.typesafe.config.ConfigFactory -import io.gearpump.streaming.appmaster.ExecutorManager.ExecutorStarted -import io.gearpump.{WorkerId, TestProbeUtil} +import org.scalatest._ + +import io.gearpump.TestProbeUtil import io.gearpump.cluster.AppMasterToWorker.ChangeExecutorResource import io.gearpump.cluster._ -import io.gearpump.cluster.appmaster.{ExecutorSystem, WorkerInfo} import io.gearpump.cluster.appmaster.ExecutorSystemScheduler.{ExecutorSystemStarted, StartExecutorSystemTimeout, StartExecutorSystems} +import io.gearpump.cluster.appmaster.{ExecutorSystem, WorkerInfo} import io.gearpump.cluster.scheduler.{Resource, ResourceRequest} +import io.gearpump.cluster.worker.WorkerId import io.gearpump.jarstore.FilePath import io.gearpump.streaming.ExecutorId import io.gearpump.streaming.ExecutorToAppMaster.RegisterExecutor -import io.gearpump.streaming.appmaster.ExecutorManager._ +import io.gearpump.streaming.appmaster.ExecutorManager.{ExecutorStarted, _} import io.gearpump.streaming.appmaster.ExecutorManagerSpec.StartExecutorActorPlease import io.gearpump.util.ActorSystemBooter.BindLifeCycle import io.gearpump.util.LogUtil -import org.scalatest._ -class ExecutorManagerSpec extends FlatSpec with Matchers with BeforeAndAfterAll { +class ExecutorManagerSpec extends FlatSpec with Matchers with BeforeAndAfterAll { implicit var system: ActorSystem = null private val LOG = LogUtil.getLogger(getClass) private val appId = 0 private val resource = Resource(10) - override def beforeAll = { + override def beforeAll(): Unit = { system = ActorSystem("test", TestUtil.DEFAULT_CONFIG) } - override def afterAll = { - system.shutdown() - system.awaitTermination() + override def afterAll(): Unit = { + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } private def startExecutorSystems = { @@ -69,17 +73,18 @@ class ExecutorManagerSpec extends FlatSpec with Matchers with BeforeAndAfterAll executor.ref ! StartExecutorActorPlease TestProbeUtil.toProps(executor) } - val executorManager = system.actorOf(Props(new ExecutorManager(userConfig, appMasterContext, executorFactory, ConfigFactory.empty, appName))) + val executorManager = system.actorOf(Props(new ExecutorManager(userConfig, appMasterContext, + executorFactory, ConfigFactory.empty, appName))) taskManager.send(executorManager, SetTaskManager(taskManager.ref)) val resourceRequest = Array(ResourceRequest(resource, WorkerId.unspecified)) - //start executors + // Starts executors taskManager.send(executorManager, StartExecutors(resourceRequest, appJar.get)) - //ask master to start executor systems + // Asks master to start executor systems import scala.concurrent.duration._ - val startExecutorSystem = master.receiveOne(5 seconds).asInstanceOf[StartExecutorSystems] + val startExecutorSystem = master.receiveOne(5.seconds).asInstanceOf[StartExecutorSystems] assert(startExecutorSystem.resources == resourceRequest) import startExecutorSystem.executorSystemConfig.{classPath, executorAkkaConfig, jar, jvmArguments, username => returnedUserName} assert(startExecutorSystem.resources == resourceRequest) @@ -94,7 +99,7 @@ class ExecutorManagerSpec extends FlatSpec with Matchers with BeforeAndAfterAll } it should "report timeout to taskManager" in { - import ExecutorManager._ + import io.gearpump.streaming.appmaster.ExecutorManager._ val (master, executor, taskManager, executorManager) = startExecutorSystems master.reply(StartExecutorSystemTimeout) taskManager.expectMsg(StartExecutorsTimeOut) @@ -110,30 +115,31 @@ class ExecutorManagerSpec extends FlatSpec with Matchers with BeforeAndAfterAll resource, workerInfo) master.reply(ExecutorSystemStarted(executorSystem, None)) import scala.concurrent.duration._ - val bindLifeWith = executorSystemDaemon.receiveOne(3 seconds).asInstanceOf[BindLifeCycle] + val bindLifeWith = executorSystemDaemon.receiveOne(3.seconds).asInstanceOf[BindLifeCycle] val proxyExecutor = bindLifeWith.actor executor.expectMsg(StartExecutorActorPlease) val executorId = 0 - //register executor - executor.send(executorManager, RegisterExecutor(proxyExecutor, executorId, resource, workerInfo)) + // Registers executor + executor.send(executorManager, RegisterExecutor(proxyExecutor, executorId, + resource, workerInfo)) taskManager.expectMsgType[ExecutorStarted] - //broad message to childs + // Broadcasts message to childs taskManager.send(executorManager, BroadCast("broadcast")) executor.expectMsg("broadcast") - //unicast + // Unicast taskManager.send(executorManager, UniCast(executorId, "unicast")) executor.expectMsg("unicast") - //update executor resource status + // Updates executor resource status val usedResource = Resource(5) executorManager ! ExecutorResourceUsageSummary(Map(executorId -> usedResource)) worker.expectMsg(ChangeExecutorResource(appId, executorId, resource - usedResource)) - //watch for executor termination + // Watches for executor termination system.stop(executor.ref) LOG.info("Shutting down executor, and wait taskManager to get notified") taskManager.expectMsg(ExecutorStopped(executorId)) diff --git a/streaming/src/test/scala/io/gearpump/streaming/appmaster/ExecutorRestartPolicySpec.scala b/streaming/src/test/scala/io/gearpump/streaming/appmaster/ExecutorRestartPolicySpec.scala index 35659d678..9d4432ad7 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/appmaster/ExecutorRestartPolicySpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/appmaster/ExecutorRestartPolicySpec.scala @@ -15,12 +15,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package io.gearpump.streaming.appmaster +import scala.concurrent.duration._ + +import org.scalatest.{Matchers, WordSpec} + import io.gearpump.streaming.executor.ExecutorRestartPolicy import io.gearpump.streaming.task.TaskId -import org.scalatest.{Matchers, WordSpec} -import scala.concurrent.duration._ class ExecutorRestartPolicySpec extends WordSpec with Matchers { @@ -29,7 +32,8 @@ class ExecutorRestartPolicySpec extends WordSpec with Matchers { val executorId1 = 1 val executorId2 = 2 val taskId = TaskId(0, 0) - val executorSupervisor = new ExecutorRestartPolicy(maxNrOfRetries = 3, withinTimeRange = 1 seconds) + val executorSupervisor = new ExecutorRestartPolicy( + maxNrOfRetries = 3, withinTimeRange = 1.seconds) executorSupervisor.addTaskToExecutor(executorId1, taskId) assert(executorSupervisor.allowRestartExecutor(executorId1)) assert(executorSupervisor.allowRestartExecutor(executorId1)) diff --git a/streaming/src/test/scala/io/gearpump/streaming/appmaster/HistoryMetricsServiceSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/appmaster/HistoryMetricsServiceSpec.scala index d023be8ba..6cd70d950 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/appmaster/HistoryMetricsServiceSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/appmaster/HistoryMetricsServiceSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,17 +18,20 @@ package io.gearpump.streaming.appmaster -import akka.actor.{Props, ActorSystem} +import scala.concurrent.Await + +import akka.actor.{ActorSystem, Props} import akka.testkit.TestProbe +import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} + import io.gearpump.cluster.ClientToMaster.QueryHistoryMetrics -import io.gearpump.cluster.MasterToClient.{HistoryMetrics, HistoryMetricsItem} +import io.gearpump.cluster.MasterToClient.HistoryMetrics import io.gearpump.cluster.TestUtil -import io.gearpump.metrics.Metrics.{Histogram, Meter, Counter} +import io.gearpump.metrics.Metrics.{Counter, Histogram, Meter} import io.gearpump.util.HistoryMetricsService -import HistoryMetricsService._ -import org.scalatest.{BeforeAndAfterEach, Matchers, FlatSpec} +import io.gearpump.util.HistoryMetricsService._ -class HistoryMetricsServiceSpec extends FlatSpec with Matchers with BeforeAndAfterEach { +class HistoryMetricsServiceSpec extends FlatSpec with Matchers with BeforeAndAfterEach { val count = 2 val intervalMs = 10 @@ -44,30 +47,30 @@ class HistoryMetricsServiceSpec extends FlatSpec with Matchers with BeforeAndAf val store = new SingleValueMetricsStore(count, intervalMs) var now = 0L - //only 1 data point will be kept in @intervalMs + // Only 1 data point will be kept in @intervalMs store.add(Counter("count", 1), now) store.add(Counter("count", 2), now) now = now + intervalMs + 1 - //only 1 data point will be kept in @intervalMs + // Only 1 data point will be kept in @intervalMs store.add(Counter("count", 3), now) store.add(Counter("count", 4), now) now = now + intervalMs + 1 - //only 1 data point will be kept in @intervalMs - //expire oldest data point, because we only keep @count records + // Only 1 data point will be kept in @intervalMs + // expire oldest data point, because we only keep @count records store.add(Counter("count", 5), now) store.add(Counter("count", 6), now) val result = store.read assert(result.size == count) - //the oldest value is expired + // The oldest value is expired assert(result.head.value.asInstanceOf[Counter].value == 3L) - //the newest value is inserted + // The newest value is inserted assert(result.last.value.asInstanceOf[Counter].value == 5L) } @@ -119,7 +122,8 @@ class HistoryMetricsServiceSpec extends FlatSpec with Matchers with BeforeAndAf assert(store.readHistory.map(_.value) == List(a)) } - "HistoryMetricsService" should "retain lastest metrics data and allow user to query metrics by path" in { + "HistoryMetricsService" should + "retain lastest metrics data and allow user to query metrics by path" in { implicit val system = ActorSystem("test", TestUtil.DEFAULT_CONFIG) val appId = 0 val service = system.actorOf(Props(new HistoryMetricsService("app0", config))) @@ -129,10 +133,10 @@ class HistoryMetricsServiceSpec extends FlatSpec with Matchers with BeforeAndAf val client = TestProbe() - // filter metrics with path "metric.counter" + // Filters metrics with path "metric.counter" client.send(service, QueryHistoryMetrics("metric.counter")) import scala.concurrent.duration._ - client.expectMsgPF(3 seconds) { + client.expectMsgPF(3.seconds) { case history: HistoryMetrics => assert(history.path == "metric.counter") val metricList = history.metrics @@ -141,9 +145,9 @@ class HistoryMetricsServiceSpec extends FlatSpec with Matchers with BeforeAndAf ) } - // filter metrics with path "metric.meter" + // Filters metrics with path "metric.meter" client.send(service, QueryHistoryMetrics("metric.meter")) - client.expectMsgPF(3 seconds) { + client.expectMsgPF(3.seconds) { case history: HistoryMetrics => assert(history.path == "metric.meter") val metricList = history.metrics @@ -152,9 +156,9 @@ class HistoryMetricsServiceSpec extends FlatSpec with Matchers with BeforeAndAf ) } - // filter metrics with path "metric.histogram" + // Filters metrics with path "metric.histogram" client.send(service, QueryHistoryMetrics("metric.histogram")) - client.expectMsgPF(3 seconds) { + client.expectMsgPF(3.seconds) { case history: HistoryMetrics => assert(history.path == "metric.histogram") val metricList = history.metrics @@ -163,10 +167,10 @@ class HistoryMetricsServiceSpec extends FlatSpec with Matchers with BeforeAndAf ) } - // filter metrics with path prefix "metric", all metrics which can + // Filters metrics with path prefix "metric", all metrics which can // match the path prefix will be retained. client.send(service, QueryHistoryMetrics("metric")) - client.expectMsgPF(3 seconds) { + client.expectMsgPF(3.seconds) { case history: HistoryMetrics => val metricList = history.metrics @@ -179,7 +183,7 @@ class HistoryMetricsServiceSpec extends FlatSpec with Matchers with BeforeAndAf case v: Counter => counterFound = true case v: Meter => meterFound = true case v: Histogram => histogramFound = true - case _ => //skip + case _ => // Skip } ) @@ -187,8 +191,7 @@ class HistoryMetricsServiceSpec extends FlatSpec with Matchers with BeforeAndAf assert(counterFound && meterFound && histogramFound) } - system.shutdown() - system.awaitTermination() - + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } } diff --git a/streaming/src/test/scala/io/gearpump/streaming/appmaster/JarSchedulerSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/appmaster/JarSchedulerSpec.scala index 12128c489..b3911964a 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/appmaster/JarSchedulerSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/appmaster/JarSchedulerSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,29 +17,31 @@ */ package io.gearpump.streaming.appmaster +import scala.concurrent.{Await, Future} + import akka.actor.ActorSystem -import com.typesafe.config.ConfigFactory -import io.gearpump.WorkerId -import io.gearpump.streaming.{ProcessorDescription, DAG} -import io.gearpump.cluster.{TestUtil, AppJar} +import org.scalatest.{Matchers, WordSpec} + import io.gearpump.cluster.scheduler.{Resource, ResourceRequest} +import io.gearpump.cluster.worker.WorkerId +import io.gearpump.cluster.{AppJar, TestUtil} import io.gearpump.jarstore.FilePath import io.gearpump.partitioner.{HashPartitioner, Partitioner} -import io.gearpump.streaming.appmaster.TaskSchedulerSpec.{TestTask2, TestTask1} +import io.gearpump.streaming.appmaster.TaskSchedulerSpec.{TestTask1, TestTask2} import io.gearpump.streaming.task.TaskId -import io.gearpump.streaming._ +import io.gearpump.streaming.{DAG, ProcessorDescription, _} import io.gearpump.util.Graph import io.gearpump.util.Graph._ -import org.scalatest.{Matchers, WordSpec} - -import scala.concurrent.{Await, Future} class JarSchedulerSpec extends WordSpec with Matchers { val mockJar1 = AppJar("jar1", FilePath("path")) val mockJar2 = AppJar("jar2", FilePath("path")) - val task1 = ProcessorDescription(id = 0, taskClass = classOf[TestTask1].getName, parallelism = 1, jar = mockJar1) - val task2 = ProcessorDescription(id = 1, taskClass = classOf[TestTask2].getName, parallelism = 1, jar = mockJar1) - val task3 = ProcessorDescription(id = 2, taskClass = classOf[TestTask2].getName, parallelism = 2, jar = mockJar2) + val task1 = ProcessorDescription(id = 0, taskClass = classOf[TestTask1].getName, parallelism = 1, + jar = mockJar1) + val task2 = ProcessorDescription(id = 1, taskClass = classOf[TestTask2].getName, parallelism = 1, + jar = mockJar1) + val task3 = ProcessorDescription(id = 2, taskClass = classOf[TestTask2].getName, parallelism = 2, + jar = mockJar2) val dag = DAG(Graph(task1 ~ Partitioner[HashPartitioner] ~> task2)) import scala.concurrent.duration._ @@ -49,37 +51,46 @@ class JarSchedulerSpec extends WordSpec with Matchers { val system = ActorSystem("JarSchedulerSpec") implicit val dispatcher = system.dispatcher val manager = new JarScheduler(0, "APP", TestUtil.DEFAULT_CONFIG, system) - manager.setDag(dag, Future{0L}) + manager.setDag(dag, Future { + 0L + }) val requests = Array(ResourceRequest(Resource(2), WorkerId.unspecified)) - val result = Await.result(manager.getRequestDetails(), 15 seconds) + val result = Await.result(manager.getResourceRequestDetails(), 15.seconds) assert(result.length == 1) assert(result.head.jar == mockJar1) assert(result.head.requests.deep == requests.deep) - val tasks = Await.result(manager.scheduleTask(mockJar1, WorkerId(0, 0L), 0, Resource(2)), 15 seconds) + val tasks = Await.result(manager.scheduleTask(mockJar1, WorkerId(0, 0L), 0, + Resource(2)), 15.seconds) assert(tasks.contains(TaskId(0, 0))) assert(tasks.contains(TaskId(1, 0))) val newDag = replaceDAG(dag, 1, task3, 1) - manager.setDag(newDag, Future{0}) - val requestDetails = Await.result(manager.getRequestDetails().map(_.sortBy(_.jar.name)), 15 seconds) + manager.setDag(newDag, Future { + 0 + }) + val requestDetails = Await.result(manager.getResourceRequestDetails(). + map(_.sortBy(_.jar.name)), 15.seconds) assert(requestDetails.length == 2) assert(requestDetails.last.jar == mockJar2) assert(requestDetails.last.requests.deep == requests.deep) - system.shutdown() - system.awaitTermination() + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } } - def replaceDAG(dag: DAG, oldProcessorId: ProcessorId, newProcessor: ProcessorDescription, newVersion: Int): DAG = { - val oldProcessorLife = LifeTime(dag.processors(oldProcessorId).life.birth, newProcessor.life.birth) + def replaceDAG( + dag: DAG, oldProcessorId: ProcessorId, newProcessor: ProcessorDescription, newVersion: Int) + : DAG = { + val oldProcessorLife = LifeTime(dag.processors(oldProcessorId).life.birth, + newProcessor.life.birth) val newProcessorMap = dag.processors ++ - Map(oldProcessorId -> dag.processors(oldProcessorId).copy(life = oldProcessorLife), - newProcessor.id -> newProcessor) + Map(oldProcessorId -> dag.processors(oldProcessorId).copy(life = oldProcessorLife), + newProcessor.id -> newProcessor) val newGraph = dag.graph.subGraph(oldProcessorId). - replaceVertex(oldProcessorId, newProcessor.id).addGraph(dag.graph) + replaceVertex(oldProcessorId, newProcessor.id).addGraph(dag.graph) new DAG(newVersion, newProcessorMap, newGraph) } } diff --git a/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskLocatorSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskLocatorSpec.scala index c55be84a8..2e07def9d 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskLocatorSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskLocatorSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,14 +18,15 @@ package io.gearpump.streaming.appmaster -import io.gearpump.WorkerId +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} + +import io.gearpump.cluster.worker.WorkerId import io.gearpump.streaming.appmaster.TaskLocator.Localities import io.gearpump.streaming.task.TaskId -import org.scalatest.{BeforeAndAfterAll, Matchers, FlatSpec} class TaskLocatorSpec extends FlatSpec with Matchers with BeforeAndAfterAll { it should "serialize/deserialize correctly" in { - val localities = new Localities(Map(WorkerId(0, 0L) -> Array(TaskId(0, 1), TaskId(1,2)))) + val localities = new Localities(Map(WorkerId(0, 0L) -> Array(TaskId(0, 1), TaskId(1, 2)))) Localities.toJson(localities) localities.localities.mapValues(_.toList) shouldBe diff --git a/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskManagerSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskManagerSpec.scala index 8105df30f..8153fceb1 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskManagerSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskManagerSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,10 +18,17 @@ package io.gearpump.streaming.appmaster +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, Future} + import akka.actor.{ActorRef, ActorSystem, Props} import akka.testkit.TestProbe +import org.mockito.Mockito._ +import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} + import io.gearpump.cluster.MasterToAppMaster.ReplayFromTimestampWindowTrailingEdge import io.gearpump.cluster.scheduler.{Resource, ResourceRequest} +import io.gearpump.cluster.worker.WorkerId import io.gearpump.cluster.{AppJar, TestUtil, UserConfig} import io.gearpump.jarstore.FilePath import io.gearpump.partitioner.{HashPartitioner, Partitioner, PartitionerDescription} @@ -39,11 +46,7 @@ import io.gearpump.streaming.{DAG, LifeTime, ProcessorDescription, ProcessorId} import io.gearpump.transport.HostPort import io.gearpump.util.Graph import io.gearpump.util.Graph._ -import io.gearpump.{WorkerId, Message, TimeStamp} -import org.mockito.Mockito._ -import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} - -import scala.concurrent.Future +import io.gearpump.{Message, TimeStamp} class TaskManagerSpec extends FlatSpec with Matchers with BeforeAndAfterEach { @@ -73,8 +76,8 @@ class TaskManagerSpec extends FlatSpec with Matchers with BeforeAndAfterEach { } override def afterEach(): Unit = { - system.shutdown() - system.awaitTermination() + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } it should "recover by requesting new executors when executor stopped unexpectedly" in { @@ -83,15 +86,18 @@ class TaskManagerSpec extends FlatSpec with Matchers with BeforeAndAfterEach { implicit val dispatcher = system.dispatcher val resourceRequest = Array(ResourceRequest(resource, workerId)) - when(scheduler.executorFailed(executorId)).thenReturn(Future{Some(ResourceRequestDetail(mockJar, resourceRequest))}) + when(scheduler.executorFailed(executorId)).thenReturn(Future { + Some(ResourceRequestDetail(mockJar, + resourceRequest)) + }) taskManager ! ExecutorStopped(executorId) - // when one executor stop, it will also trigger the recovery by restart + // When one executor stop, it will also trigger the recovery by restart // existing executors executorManager.expectMsg(BroadCast(RestartTasks(dagVersion))) - // ask for new executors + // Asks for new executors val returned = executorManager.receiveN(1).head.asInstanceOf[StartExecutors] assert(returned.resources.deep == resourceRequest.deep) executorManager.reply(StartExecutorsTimeOut) @@ -110,7 +116,7 @@ class TaskManagerSpec extends FlatSpec with Matchers with BeforeAndAfterEach { executorManager.expectMsg(BroadCast(RestartTasks(dagVersion))) } - import TaskManager.TaskChangeRegistry + import io.gearpump.streaming.appmaster.TaskManager.TaskChangeRegistry "TaskChangeRegistry" should "track all modified task registration" in { val tasks = List(TaskId(0, 0), TaskId(0, 1)) val registry = new TaskChangeRegistry(tasks) @@ -155,24 +161,28 @@ class TaskManagerSpec extends FlatSpec with Matchers with BeforeAndAfterEach { val dagManager = TestProbe() val taskManager = system.actorOf( - Props(new TaskManager(appId, dagManager.ref, scheduler, executorManager.ref, clockService.ref, appMaster.ref, "appName"))) + Props(new TaskManager(appId, dagManager.ref, scheduler, executorManager.ref, clockService.ref, + appMaster.ref, "appName"))) dagManager.expectMsgType[WatchChange] executorManager.expectMsgType[SetTaskManager] - // step1: first transition from Unitialized to ApplicationReady + // Step1: first transition from Unitialized to ApplicationReady executorManager.expectMsgType[ExecutorResourceUsageSummary] dagManager.expectMsgType[NewDAGDeployed] - // step2: Get Additional Resource Request - when(scheduler.getRequestDetails()) - .thenReturn(Future{Array(ResourceRequestDetail(mockJar, Array(ResourceRequest(resource, WorkerId.unspecified))))}) + // Step2: Get Additional Resource Request + when(scheduler.getResourceRequestDetails()) + .thenReturn(Future { + Array(ResourceRequestDetail(mockJar, Array(ResourceRequest(resource, + WorkerId.unspecified)))) + }) - // step3: DAG changed. Start transit from ApplicationReady -> DynamicDAG + // Step3: DAG changed. Start transit from ApplicationReady -> DynamicDAG dagManager.expectMsg(GetLatestDAG) dagManager.reply(LatestDAG(dag)) - // step4: Start remote Executors. + // Step4: Start remote Executors. // received Broadcast executorManager.expectMsg(BroadCast(StartDynamicDag(dag.version))) executorManager.expectMsgType[StartExecutors] @@ -180,10 +190,10 @@ class TaskManagerSpec extends FlatSpec with Matchers with BeforeAndAfterEach { when(scheduler.scheduleTask(mockJar, workerId, executorId, resource)) .thenReturn(Future(List(TaskId(0, 0), TaskId(1, 0)))) - // step5: Executor is started. + // Step5: Executor is started. executorManager.reply(ExecutorStarted(executorId, resource, workerId, Some(mockJar))) - // step6: Prepare to start Task. First GetTaskLaunchData. + // Step6: Prepare to start Task. First GetTaskLaunchData. val taskLaunchData: PartialFunction[Any, TaskLaunchData] = { case GetTaskLaunchData(_, 0, executorStarted) => task1LaunchData.copy(context = executorStarted) @@ -197,18 +207,17 @@ class TaskManagerSpec extends FlatSpec with Matchers with BeforeAndAfterEach { val launchData2 = dagManager.expectMsgPF()(taskLaunchData) dagManager.reply(launchData2) - // step7: Launch Task + // Step7: Launch Task val launchTaskMatch: PartialFunction[Any, RegisterTask] = { case UniCast(executorId, launch: LaunchTasks) => - Console.println("Launch Task " + launch.processorDescription.id) RegisterTask(launch.taskId.head, executorId, HostPort("127.0.0.1:3000")) } - // taskmanager should return the latest start clock to task(0,0) + // Taskmanager should return the latest start clock to task(0,0) clockService.expectMsg(GetStartClock) clockService.reply(StartClock(0)) - // step8: Task is started. registerTask. + // Step8: Task is started. registerTask. val registerTask1 = executorManager.expectMsgPF()(launchTaskMatch) taskManager.tell(registerTask1, executor.ref) executor.expectMsgType[TaskRegistered] @@ -217,53 +226,51 @@ class TaskManagerSpec extends FlatSpec with Matchers with BeforeAndAfterEach { taskManager.tell(registerTask2, executor.ref) executor.expectMsgType[TaskRegistered] - // step9: start broadcasting TaskLocations. + // Step9: start broadcasting TaskLocations. import scala.concurrent.duration._ - assert(executorManager.expectMsgPF(5 seconds) { + assert(executorManager.expectMsgPF(5.seconds) { case BroadCast(startAllTasks) => startAllTasks.isInstanceOf[TaskLocationsReady] }) - //step10: Executor confirm it has received TaskLocationsReceived(version, executorId) + // Step10: Executor confirm it has received TaskLocationsReceived(version, executorId) taskManager.tell(TaskLocationsReceived(dag.version, executorId), executor.ref) - - // step11: Tell ClockService to update DAG. + // Step11: Tell ClockService to update DAG. clockService.expectMsgType[ChangeToNewDAG] clockService.reply(ChangeToNewDAGSuccess(Map.empty[ProcessorId, TimeStamp])) - - //step12: start all tasks + // Step12: start all tasks import scala.concurrent.duration._ - assert(executorManager.expectMsgPF(5 seconds) { + assert(executorManager.expectMsgPF(5.seconds) { case BroadCast(startAllTasks) => startAllTasks.isInstanceOf[StartAllTasks] }) - // step13, Tell executor Manager the updated usage status of executors. + // Step13, Tell executor Manager the updated usage status of executors. executorManager.expectMsgType[ExecutorResourceUsageSummary] - // step14: transition from DynamicDAG to ApplicationReady + // Step14: transition from DynamicDAG to ApplicationReady Env(executorManager, clockService, appMaster, executor, taskManager, scheduler) } } object TaskManagerSpec { case class Env( - executorManager: TestProbe, - clockService: TestProbe, - appMaster: TestProbe, - executor: TestProbe, - taskManager: ActorRef, - scheduler: JarScheduler) - - class Task1(taskContext : TaskContext, userConf : UserConfig) + executorManager: TestProbe, + clockService: TestProbe, + appMaster: TestProbe, + executor: TestProbe, + taskManager: ActorRef, + scheduler: JarScheduler) + + class Task1(taskContext: TaskContext, userConf: UserConfig) extends Task(taskContext, userConf) { - override def onStart(startTime: StartTime): Unit = ??? - override def onNext(msg: Message): Unit = ??? + override def onStart(startTime: StartTime): Unit = {} + override def onNext(msg: Message): Unit = {} } - class Task2 (taskContext : TaskContext, userConf : UserConfig) + class Task2(taskContext: TaskContext, userConf: UserConfig) extends Task(taskContext, userConf) { - override def onStart(startTime: StartTime): Unit = ??? - override def onNext(msg: Message): Unit = ??? + override def onStart(startTime: StartTime): Unit = {} + override def onNext(msg: Message): Unit = {} } } \ No newline at end of file diff --git a/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskRegistrySpec.scala b/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskRegistrySpec.scala index ecac824d8..e8417ea68 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskRegistrySpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskRegistrySpec.scala @@ -18,12 +18,12 @@ package io.gearpump.streaming.appmaster -import io.gearpump.streaming.appmaster.TaskRegistry.{Reject, Accept} +import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} + import io.gearpump.cluster.scheduler.Resource import io.gearpump.streaming.appmaster.TaskRegistry.{Accept, Reject, TaskLocation, TaskLocations} import io.gearpump.streaming.task.TaskId import io.gearpump.transport.HostPort -import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers} class TaskRegistrySpec extends FlatSpec with Matchers with BeforeAndAfterEach { it should "maintain registered tasks" in { diff --git a/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskSchedulerSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskSchedulerSpec.scala index d2373ea77..2c641339f 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskSchedulerSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/appmaster/TaskSchedulerSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,23 +17,22 @@ */ package io.gearpump.streaming.appmaster +import scala.collection.mutable.ArrayBuffer + import com.typesafe.config.ConfigFactory -import io.gearpump.streaming.Constants -import io.gearpump.streaming.appmaster.TaskLocator.Localities -import io.gearpump.streaming.task.{StartTime, TaskContext, TaskId} -import io.gearpump.{WorkerId, Message} +import org.scalatest.{Matchers, WordSpec} + +import io.gearpump.Message import io.gearpump.cluster.scheduler.{Relaxation, Resource, ResourceRequest} -import io.gearpump.cluster.{TestUtil, ClusterConfig, UserConfig} +import io.gearpump.cluster.worker.WorkerId +import io.gearpump.cluster.{TestUtil, UserConfig} import io.gearpump.partitioner.{HashPartitioner, Partitioner} import io.gearpump.streaming.appmaster.TaskLocator.Localities import io.gearpump.streaming.appmaster.TaskSchedulerSpec.{TestTask1, TestTask2} import io.gearpump.streaming.task.{StartTime, Task, TaskContext, TaskId} -import io.gearpump.streaming.{DAG, ProcessorDescription} +import io.gearpump.streaming.{Constants, DAG, ProcessorDescription} import io.gearpump.util.Graph import io.gearpump.util.Graph._ -import org.scalatest.{Matchers, WordSpec} - -import scala.collection.mutable.ArrayBuffer class TaskSchedulerSpec extends WordSpec with Matchers { val task1 = ProcessorDescription(id = 0, taskClass = classOf[TestTask1].getName, parallelism = 4) @@ -47,19 +46,19 @@ class TaskSchedulerSpec extends WordSpec with Matchers { "schedule tasks on different workers properly according user's configuration" in { val localities = Localities( - Map(WorkerId(1, 0L) -> Array(TaskId(0,0), TaskId(0,1), TaskId(1,0), TaskId(1,1)), - WorkerId(2, 0L) -> Array(TaskId(0,2), TaskId(0,3)) - )) + Map(WorkerId(1, 0L) -> Array(TaskId(0, 0), TaskId(0, 1), TaskId(1, 0), TaskId(1, 1)), + WorkerId(2, 0L) -> Array(TaskId(0, 2), TaskId(0, 3)) + )) val localityConfig = ConfigFactory.parseString(Localities.toJson(localities)) - import Constants.GEARPUMP_STREAMING_LOCALITIES + import io.gearpump.streaming.Constants.GEARPUMP_STREAMING_LOCALITIES val appName = "app" val taskScheduler = new TaskSchedulerImpl(appId = 0, appName, config.withValue(s"$GEARPUMP_STREAMING_LOCALITIES.$appName", localityConfig.root)) val expectedRequests = - Array( ResourceRequest(Resource(4), WorkerId(1, 0L), relaxation = Relaxation.SPECIFICWORKER), + Array(ResourceRequest(Resource(4), WorkerId(1, 0L), relaxation = Relaxation.SPECIFICWORKER), ResourceRequest(Resource(2), WorkerId(2, 0L), relaxation = Relaxation.SPECIFICWORKER)) taskScheduler.setDAG(dag) @@ -71,28 +70,32 @@ class TaskSchedulerSpec extends WordSpec with Matchers { val tasksOnWorker1 = ArrayBuffer[Int]() val tasksOnWorker2 = ArrayBuffer[Int]() for (i <- 0 until 4) { - tasksOnWorker1.append(taskScheduler.schedule(WorkerId(1, 0L), executorId = 0, Resource(1)).head.processorId) + tasksOnWorker1.append(taskScheduler.schedule(WorkerId(1, 0L), + executorId = 0, Resource(1)).head.processorId) } for (i <- 0 until 2) { - tasksOnWorker2.append(taskScheduler.schedule(WorkerId(2, 0L), executorId = 1, Resource(1)).head.processorId) + tasksOnWorker2.append(taskScheduler.schedule(WorkerId(2, 0L), executorId = 1, + Resource(1)).head.processorId) } - //allocate more resource, and no tasks to launch - assert(taskScheduler.schedule(WorkerId(3, 0L), executorId = 3, Resource(1)) == List.empty[TaskId]) + // Allocates more resource, and no tasks to launch + assert(taskScheduler.schedule(WorkerId(3, 0L), executorId = 3, + Resource(1)) == List.empty[TaskId]) - //on worker1, executor 0 + // On worker1, executor 0 assert(tasksOnWorker1.sorted.sameElements(Array(0, 0, 1, 1))) - //on worker2, executor 1, Task(0, 0), Task(0, 1) + // On worker2, executor 1, Task(0, 0), Task(0, 1) assert(tasksOnWorker2.sorted.sameElements(Array(0, 0))) val rescheduledResources = taskScheduler.executorFailed(executorId = 1) - assert(rescheduledResources.sameElements(Array(ResourceRequest(Resource(2), WorkerId.unspecified, relaxation = Relaxation.ONEWORKER)))) + assert(rescheduledResources.sameElements(Array(ResourceRequest(Resource(2), + WorkerId.unspecified, relaxation = Relaxation.ONEWORKER)))) val launchedTask = taskScheduler.schedule(WorkerId(3, 0L), executorId = 3, Resource(2)) - //start the failed 2 tasks Task(0, 0) and Task(0, 1) + // Starts the failed 2 tasks Task(0, 0) and Task(0, 1) assert(launchedTask.length == 2) } @@ -101,7 +104,7 @@ class TaskSchedulerSpec extends WordSpec with Matchers { val taskScheduler = new TaskSchedulerImpl(appId = 0, appName, config) val expectedRequests = - Array( ResourceRequest(Resource(4), WorkerId(1, 0L), relaxation = Relaxation.SPECIFICWORKER), + Array(ResourceRequest(Resource(4), WorkerId(1, 0L), relaxation = Relaxation.SPECIFICWORKER), ResourceRequest(Resource(2), WorkerId(2, 0L), relaxation = Relaxation.SPECIFICWORKER)) taskScheduler.setDAG(dag) @@ -112,16 +115,16 @@ class TaskSchedulerSpec extends WordSpec with Matchers { } } -object TaskSchedulerSpec{ - class TestTask1(taskContext : TaskContext, userConf : UserConfig) - extends Task(taskContext, userConf) { - override def onStart(startTime: StartTime): Unit = ??? - override def onNext(msg: Message): Unit = ??? +object TaskSchedulerSpec { + class TestTask1(taskContext: TaskContext, userConf: UserConfig) + extends Task(taskContext, userConf) { + override def onStart(startTime: StartTime): Unit = Unit + override def onNext(msg: Message): Unit = Unit } - class TestTask2(taskContext : TaskContext, userConf : UserConfig) - extends Task(taskContext, userConf) { - override def onStart(startTime: StartTime): Unit = ??? - override def onNext(msg: Message): Unit = ??? + class TestTask2(taskContext: TaskContext, userConf: UserConfig) + extends Task(taskContext, userConf) { + override def onStart(startTime: StartTime): Unit = Unit + override def onNext(msg: Message): Unit = Unit } } diff --git a/streaming/src/test/scala/io/gearpump/streaming/dsl/StreamAppSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/dsl/StreamAppSpec.scala index 89d2d648a..132d46c46 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/dsl/StreamAppSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/dsl/StreamAppSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,28 +18,30 @@ package io.gearpump.streaming.dsl +import scala.concurrent.Await +import scala.concurrent.duration.Duration + import akka.actor.ActorSystem -import io.gearpump.streaming.dsl.plan.OpTranslator.SourceTask -import io.gearpump.cluster.TestUtil -import io.gearpump.cluster.client.ClientContext -import io.gearpump.streaming.dsl.plan.OpTranslator._ import org.mockito.Mockito.when import org.scalatest._ import org.scalatest.mock.MockitoSugar -class StreamAppSpec extends FlatSpec with Matchers with BeforeAndAfterAll with MockitoSugar { + +import io.gearpump.cluster.TestUtil +import io.gearpump.cluster.client.ClientContext +import io.gearpump.streaming.dsl.plan.OpTranslator.SourceTask +class StreamAppSpec extends FlatSpec with Matchers with BeforeAndAfterAll with MockitoSugar { implicit var system: ActorSystem = null - override def beforeAll: Unit = { - system = ActorSystem("test", TestUtil.DEFAULT_CONFIG) + override def beforeAll(): Unit = { + system = ActorSystem("test", TestUtil.DEFAULT_CONFIG) } - override def afterAll: Unit = { - system.shutdown() - system.awaitTermination() + override def afterAll(): Unit = { + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } - it should "be able to generate multiple new streams" in { val context: ClientContext = mock[ClientContext] when(context.system).thenReturn(system) @@ -57,7 +59,7 @@ class StreamAppSpec extends FlatSpec with Matchers with BeforeAndAfterAll with val app = StreamApp("dsl", context) val parallism = 3 - app.source(List("A","B","C"), parallism, "").flatMap(Array(_)).reduce(_+_) + app.source(List("A", "B", "C"), parallism, "").flatMap(Array(_)).reduce(_ + _) val task = app.plan.dag.vertices.iterator.next() assert(task.taskClass == classOf[SourceTask[_, _]].getName) assert(task.parallelism == parallism) @@ -72,7 +74,7 @@ class StreamAppSpec extends FlatSpec with Matchers with BeforeAndAfterAll with "1", "2" ) - val producer = app.source(list, 1, "producer").flatMap(Array(_)).reduce(_+_) + val producer = app.source(list, 1, "producer").flatMap(Array(_)).reduce(_ + _) val task = app.plan.dag.vertices.iterator.next() /* val task = app.plan.dag.vertices.iterator.map(desc => { diff --git a/streaming/src/test/scala/io/gearpump/streaming/dsl/StreamSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/dsl/StreamSpec.scala index c25acb1c1..c8a44f65e 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/dsl/StreamSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/dsl/StreamSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,11 +18,18 @@ package io.gearpump.streaming.dsl +import scala.concurrent.Await +import scala.concurrent.duration.Duration +import scala.util.{Either, Left, Right} + import akka.actor._ -import io.gearpump.streaming.task.{StartTime, TaskContext} +import org.mockito.Mockito.when +import org.scalatest._ +import org.scalatest.mock.MockitoSugar + import io.gearpump.Message -import io.gearpump.cluster.{TestUtil, UserConfig} import io.gearpump.cluster.client.ClientContext +import io.gearpump.cluster.{TestUtil, UserConfig} import io.gearpump.partitioner.{CoLocationPartitioner, HashPartitioner} import io.gearpump.streaming.dsl.StreamSpec.Join import io.gearpump.streaming.dsl.partitioner.GroupByPartitioner @@ -30,23 +37,18 @@ import io.gearpump.streaming.dsl.plan.OpTranslator._ import io.gearpump.streaming.task.{StartTime, Task, TaskContext} import io.gearpump.util.Graph import io.gearpump.util.Graph._ -import org.mockito.Mockito.when -import org.scalatest._ -import org.scalatest.mock.MockitoSugar -import scala.util.{Either, Left, Right} - -class StreamSpec extends FlatSpec with Matchers with BeforeAndAfterAll with MockitoSugar { +class StreamSpec extends FlatSpec with Matchers with BeforeAndAfterAll with MockitoSugar { implicit var system: ActorSystem = null - override def beforeAll: Unit = { - system = ActorSystem("test", TestUtil.DEFAULT_CONFIG) + override def beforeAll(): Unit = { + system = ActorSystem("test", TestUtil.DEFAULT_CONFIG) } override def afterAll(): Unit = { - system.shutdown() - system.awaitTermination() + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } it should "translate the DSL to a DAG" in { @@ -55,7 +57,7 @@ class StreamSpec extends FlatSpec with Matchers with BeforeAndAfterAll with Mo val app = StreamApp("dsl", context) - val data = + val data = """ five four three two one five four three two @@ -75,7 +77,9 @@ class StreamSpec extends FlatSpec with Matchers with BeforeAndAfterAll with Mo val appDescription = app.plan - val dagTopology = appDescription.dag.mapVertex(_.taskClass).mapEdge((node1, edge, node2) => edge.partitionerFactory.partitioner.getClass.getName) + val dagTopology = appDescription.dag.mapVertex(_.taskClass).mapEdge { (node1, edge, node2) => + edge.partitionerFactory.partitioner.getClass.getName + } val expectedDagTopology = getExpectedDagTopology assert(dagTopology.vertices.toSet.equals(expectedDagTopology.vertices.toSet)) @@ -102,14 +106,14 @@ class StreamSpec extends FlatSpec with Matchers with BeforeAndAfterAll with Mo object StreamSpec { - class Join(taskContext : TaskContext, userConf : UserConfig) extends Task(taskContext, userConf) { + class Join(taskContext: TaskContext, userConf: UserConfig) extends Task(taskContext, userConf) { var query: String = null override def onStart(startTime: StartTime): Unit = {} override def onNext(msg: Message): Unit = { msg.msg match { - case Left(wordCount: (String, Int)) => + case Left(wordCount: (String @unchecked, Int @unchecked)) => if (query != null && wordCount._1 == query) { taskContext.output(new Message(wordCount)) } diff --git a/streaming/src/test/scala/io/gearpump/streaming/dsl/partitioner/GroupByPartitionerSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/dsl/partitioner/GroupByPartitionerSpec.scala index fa2c8cfe7..03dd2420b 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/dsl/partitioner/GroupByPartitionerSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/dsl/partitioner/GroupByPartitionerSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,11 +18,12 @@ package io.gearpump.streaming.dsl.partitioner +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} + import io.gearpump.Message import io.gearpump.streaming.dsl.partitioner.GroupByPartitionerSpec.People -import org.scalatest.{BeforeAndAfterAll, Matchers, FlatSpec} -class GroupByPartitionerSpec extends FlatSpec with Matchers with BeforeAndAfterAll { +class GroupByPartitionerSpec extends FlatSpec with Matchers with BeforeAndAfterAll { it should "use the outpout of groupBy function to do partition" in { val mark = People("Mark", "male") val tom = People("Tom", "male") diff --git a/streaming/src/test/scala/io/gearpump/streaming/dsl/plan/OpTranslatorSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/dsl/plan/OpTranslatorSpec.scala index 13af9c207..62a0f9594 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/dsl/plan/OpTranslatorSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/dsl/plan/OpTranslatorSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,7 +18,15 @@ package io.gearpump.streaming.dsl.plan +import scala.concurrent.Await +import scala.concurrent.duration.Duration + import akka.actor.ActorSystem +import org.mockito.ArgumentCaptor +import org.mockito.Matchers._ +import org.mockito.Mockito._ +import org.scalatest._ + import io.gearpump.Message import io.gearpump.cluster.{TestUtil, UserConfig} import io.gearpump.streaming.Constants._ @@ -26,29 +34,25 @@ import io.gearpump.streaming.MockUtil import io.gearpump.streaming.dsl.CollectionDataSource import io.gearpump.streaming.dsl.plan.OpTranslator._ import io.gearpump.streaming.task.StartTime -import org.mockito.ArgumentCaptor -import org.mockito.Matchers._ -import org.mockito.Mockito._ -import org.scalatest._ -class OpTranslatorSpec extends FlatSpec with Matchers with BeforeAndAfterAll { +class OpTranslatorSpec extends FlatSpec with Matchers with BeforeAndAfterAll { "andThen" should "chain multiple single input function" in { val dummy = new DummyInputFunction[String] val split = new FlatMapFunction[String, String](line => line.split("\\s"), "split") - val filter = new FlatMapFunction[String, String](word => if (word.isEmpty) None else Some(word), "filter") + val filter = new FlatMapFunction[String, String](word => + if (word.isEmpty) None else Some(word), "filter") val map = new FlatMapFunction[String, Int](word => Some(1), "map") - val sum = new ReduceFunction[Int]({ (left, right) => left + right}, "sum") + val sum = new ReduceFunction[Int]({ (left, right) => left + right }, "sum") val all = dummy.andThen(split).andThen(filter).andThen(map).andThen(sum) - assert(all.description == "split.filter.map.sum") - val data = + val data = """ five four three two one five four three two @@ -67,22 +71,25 @@ class OpTranslatorSpec extends FlatSpec with Matchers with BeforeAndAfterAll { val conf = UserConfig.empty val data = "one two three".split("\\s") - //source with no transformer - val source = new SourceTask[String, String](new CollectionDataSource[String](data), None, taskContext, conf) + // Source with no transformer + val source = new SourceTask[String, String](new CollectionDataSource[String](data), None, + taskContext, conf) source.onStart(StartTime(0)) source.onNext(Message("next")) verify(taskContext, times(1)).output(anyObject()) - //source with transformer + // Source with transformer val anotherTaskContext = MockUtil.mockTaskContext val double = new FlatMapFunction[String, String](word => List(word, word), "double") - val another = new SourceTask(new CollectionDataSource[String](data), Some(double), anotherTaskContext, conf) + val another = new SourceTask(new CollectionDataSource[String](data), Some(double), + anotherTaskContext, conf) another.onStart(StartTime(0)) another.onNext(Message("next")) verify(anotherTaskContext, times(2)).output(anyObject()) } - "GroupByTask" should "group input by groupBy Function and apply attached operator for each group" in { + "GroupByTask" should "group input by groupBy Function and " + + "apply attached operator for each group" in { val data = "1 2 2 3 3 3" @@ -92,32 +99,33 @@ class OpTranslatorSpec extends FlatSpec with Matchers with BeforeAndAfterAll { left + right }, "concat") - implicit val system = ActorSystem("test", TestUtil.DEFAULT_CONFIG) - val config = UserConfig.empty.withValue[SingleInputFunction[String, String]](GEARPUMP_STREAMING_OPERATOR, concat) + implicit val system = ActorSystem("test", TestUtil.DEFAULT_CONFIG) + val config = UserConfig.empty.withValue[SingleInputFunction[String, String]]( + GEARPUMP_STREAMING_OPERATOR, concat) val taskContext = MockUtil.mockTaskContext val task = new GroupByTask[String, String, String](input => input, taskContext, config) task.onStart(StartTime(0)) - val peopleCaptor = ArgumentCaptor.forClass(classOf[Message]); + val peopleCaptor = ArgumentCaptor.forClass(classOf[Message]) data.split("\\s+").foreach { word => task.onNext(Message(word)) } - verify(taskContext, times(6)).output(peopleCaptor.capture()); + verify(taskContext, times(6)).output(peopleCaptor.capture()) import scala.collection.JavaConverters._ val values = peopleCaptor.getAllValues().asScala.map(input => input.msg.asInstanceOf[String]) assert(values.mkString(",") == "1,2,22,3,33,333") - system.shutdown - system.awaitTermination() + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } "MergeTask" should "accept two stream and apply the attached operator" in { - //source with transformer + // Source with transformer val taskContext = MockUtil.mockTaskContext val conf = UserConfig.empty val double = new FlatMapFunction[String, String](word => List(word, word), "double") @@ -126,7 +134,7 @@ class OpTranslatorSpec extends FlatSpec with Matchers with BeforeAndAfterAll { val data = "1 2 2 3 3 3".split("\\s+") - data.foreach{input => + data.foreach { input => task.onNext(Message(input)) } diff --git a/streaming/src/test/scala/io/gearpump/streaming/executor/ExecutorSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/executor/ExecutorSpec.scala index 151a188cf..1d947763c 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/executor/ExecutorSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/executor/ExecutorSpec.scala @@ -1,27 +1,34 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.gearpump.streaming.executor +import scala.concurrent.Await +import scala.concurrent.duration.Duration + import akka.actor.{ActorSystem, Props} import akka.testkit.TestProbe -import io.gearpump.WorkerId +import org.mockito.Matchers._ +import org.mockito.Mockito.{times, _} +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} + import io.gearpump.cluster.appmaster.WorkerInfo import io.gearpump.cluster.scheduler.Resource +import io.gearpump.cluster.worker.WorkerId import io.gearpump.cluster.{ExecutorContext, TestUtil, UserConfig} import io.gearpump.streaming.AppMasterToExecutor._ import io.gearpump.streaming.ExecutorToAppMaster.RegisterTask @@ -30,12 +37,6 @@ import io.gearpump.streaming.executor.TaskLauncherSpec.MockTask import io.gearpump.streaming.task.{Subscriber, TaskId} import io.gearpump.streaming.{LifeTime, ProcessorDescription} import io.gearpump.transport.HostPort -import org.mockito.Matchers._ -import org.mockito.Mockito.{times, _} -import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} - -import scala.language.postfixOps - class ExecutorSpec extends FlatSpec with Matchers with BeforeAndAfterAll { val appId = 0 @@ -51,22 +52,25 @@ class ExecutorSpec extends FlatSpec with Matchers with BeforeAndAfterAll { } override def afterAll(): Unit = { - system.shutdown() - system.awaitTermination() + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } it should "call launcher to launch task" in { val worker = TestProbe() val workerInfo = WorkerInfo(workerId, worker.ref) - val executorContext = ExecutorContext(executorId, workerInfo, appId, "app", appMaster.ref, Resource(2)) + val executorContext = ExecutorContext(executorId, workerInfo, appId, "app", + appMaster.ref, Resource(2)) val taskLauncher = mock(classOf[ITaskLauncher]) val executor = system.actorOf(Props(new Executor(executorContext, userConf, taskLauncher))) - val processor = ProcessorDescription(id = 0, taskClass = classOf[MockTask].getName, parallelism = 2) + val processor = ProcessorDescription(id = 0, taskClass = classOf[MockTask].getName, + parallelism = 2) val taskIds = List(TaskId(0, 0), TaskId(0, 1)) - val launchTasks = LaunchTasks(taskIds, dagVersion = 0, processor, List.empty[Subscriber]) + val launchTasks = LaunchTasks(taskIds, dagVersion = 0, processor, List.empty[Subscriber]) val task = TestProbe() - when(taskLauncher.launch(any(), any(), any(), any(), any())).thenReturn(taskIds.map((_, task.ref)).toMap) + when(taskLauncher.launch(any(), any(), any(), any(), any())) + .thenReturn(taskIds.map((_, task.ref)).toMap) val client = TestProbe() client.send(executor, launchTasks) @@ -91,7 +95,8 @@ class ExecutorSpec extends FlatSpec with Matchers with BeforeAndAfterAll { task.expectMsgType[StartTask] task.expectMsgType[StartTask] - val changeTasks = ChangeTasks(taskIds, dagVersion = 1, life = LifeTime(0, Long.MaxValue), List.empty[Subscriber]) + val changeTasks = ChangeTasks(taskIds, dagVersion = 1, life = LifeTime(0, Long.MaxValue), + List.empty[Subscriber]) client.send(executor, changeTasks) client.expectMsgType[TasksChanged] diff --git a/streaming/src/test/scala/io/gearpump/streaming/executor/TaskArgumentStoreSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/executor/TaskArgumentStoreSpec.scala index d220e601f..c97ae4c43 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/executor/TaskArgumentStoreSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/executor/TaskArgumentStoreSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,13 +17,13 @@ */ package io.gearpump.streaming.executor -import io.gearpump.streaming.executor.TaskLauncher.TaskArgument import org.scalatest._ -import io.gearpump.streaming.executor.Executor.{TaskArgumentStore} -import scala.language.postfixOps + +import io.gearpump.streaming.executor.Executor.TaskArgumentStore +import io.gearpump.streaming.executor.TaskLauncher.TaskArgument import io.gearpump.streaming.task.TaskId -class TaskArgumentStoreSpec extends FlatSpec with Matchers with BeforeAndAfterEach { +class TaskArgumentStoreSpec extends FlatSpec with Matchers with BeforeAndAfterEach { it should "retain all history of taskArgument" in { val version0 = TaskArgument(0, null, null) val version2 = version0.copy(dagVersion = 2) @@ -32,15 +32,14 @@ class TaskArgumentStoreSpec extends FlatSpec with Matchers with BeforeAndAfterE store.add(task, version0) store.add(task, version2) - // we should return a version which is same or older than expected version + // Should return a version which is same or older than expected version assert(store.get(dagVersion = 1, task) == Some(version0)) assert(store.get(dagVersion = 0, task) == Some(version0)) assert(store.get(dagVersion = 2, task) == Some(version2)) - store.removeObsoleteVersion + store.removeObsoleteVersion() assert(store.get(dagVersion = 1, task) == None) assert(store.get(dagVersion = 0, task) == None) assert(store.get(dagVersion = 2, task) == Some(version2)) } - } diff --git a/streaming/src/test/scala/io/gearpump/streaming/executor/TaskLauncherSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/executor/TaskLauncherSpec.scala index f9decf557..c135c5bb9 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/executor/TaskLauncherSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/executor/TaskLauncherSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,19 +17,21 @@ */ package io.gearpump.streaming.executor +import scala.concurrent.Await +import scala.concurrent.duration.Duration + import akka.actor.{Actor, ActorSystem} import akka.testkit.TestProbe +import org.scalatest._ + import io.gearpump.cluster.{TestUtil, UserConfig} import io.gearpump.serializer.SerializationFramework import io.gearpump.streaming.ProcessorDescription import io.gearpump.streaming.executor.TaskLauncher.TaskArgument import io.gearpump.streaming.executor.TaskLauncherSpec.{MockTask, MockTaskActor} import io.gearpump.streaming.task.{Task, TaskContext, TaskContextData, TaskId, TaskWrapper} -import org.scalatest._ - -import scala.language.postfixOps -class TaskLauncherSpec extends FlatSpec with Matchers with BeforeAndAfterAll { +class TaskLauncherSpec extends FlatSpec with Matchers with BeforeAndAfterAll { val appId = 0 val executorId = 0 var appMaster: TestProbe = null @@ -42,17 +44,20 @@ class TaskLauncherSpec extends FlatSpec with Matchers with BeforeAndAfterAll { } override def afterAll(): Unit = { - system.shutdown() - system.awaitTermination() + system.terminate() + Await.result(system.whenTerminated, Duration.Inf) } it should "able to launch tasks" in { - val launcher = new TaskLauncher(appId, "app", executorId, appMaster.ref, userConf, classOf[MockTaskActor]) + val launcher = new TaskLauncher(appId, "app", executorId, appMaster.ref, + userConf, classOf[MockTaskActor]) val taskIds = List(TaskId(0, 0), TaskId(0, 1)) - val processor = ProcessorDescription(id = 0, taskClass = classOf[MockTask].getName, parallelism = 2) + val processor = ProcessorDescription(id = 0, taskClass = classOf[MockTask].getName, + parallelism = 2) val argument = TaskArgument(0, processor, null) - val tasks = launcher.launch(taskIds, argument, system, null, "gearpump.shared-thread-pool-dispatcher") + val tasks = launcher.launch(taskIds, argument, system, null, + "gearpump.shared-thread-pool-dispatcher") tasks.keys.toSet shouldBe taskIds.toSet } } @@ -67,6 +72,7 @@ object TaskLauncherSpec { def receive: Receive = null } - class MockTask(taskContext : TaskContext, userConf : UserConfig) extends Task(taskContext, userConf) { + class MockTask(taskContext: TaskContext, userConf: UserConfig) + extends Task(taskContext, userConf) { } } diff --git a/streaming/src/test/scala/io/gearpump/streaming/metrics/ProcessorAggregatorSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/metrics/ProcessorAggregatorSpec.scala index c147a4b80..da1ca9f7b 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/metrics/ProcessorAggregatorSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/metrics/ProcessorAggregatorSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,26 +18,27 @@ package io.gearpump.streaming.metrics +import scala.collection.JavaConverters._ +import scala.util.Random + +import org.scalatest.{FlatSpec, Matchers} + import io.gearpump.cluster.ClientToMaster.ReadOption import io.gearpump.cluster.MasterToClient.HistoryMetricsItem -import io.gearpump.metrics.Metrics.{Gauge, Meter, Histogram} -import io.gearpump.streaming.metrics.ProcessorAggregator.{AggregatorFactory, MeterAggregator, HistogramAggregator, MultiLayerMap} +import io.gearpump.metrics.Metrics.{Gauge, Histogram, Meter} +import io.gearpump.streaming.metrics.ProcessorAggregator.{AggregatorFactory, HistogramAggregator, MeterAggregator, MultiLayerMap} import io.gearpump.streaming.task.TaskId import io.gearpump.util.HistoryMetricsService.HistoryMetricsConfig -import org.scalatest.{Matchers, FlatSpec} -import scala.collection.JavaConverters._ -import scala.util.Random class ProcessorAggregatorSpec extends FlatSpec with Matchers { - "MultiLayerMap" should "maintain multiple layers HashMap" in { val layers = 3 val map = new MultiLayerMap[String](layers) assert(map.get(layer = 0, "key") == null) - // illegal, handle safely + // Illegal, handle safely assert(map.get(layer = 10, "key") == null) map.put(layer = 0, "key", "value") @@ -47,7 +48,7 @@ class ProcessorAggregatorSpec extends FlatSpec with Matchers { map.put(layer = 2, "key3", "value3") map.put(layer = 2, "key4", "value4") - // illegal, should be ignored + // Illegal, should be ignored map.put(layer = 4, "key5", "value5") assert(map.size == 4) @@ -69,15 +70,15 @@ class ProcessorAggregatorSpec extends FlatSpec with Matchers { val result = aggregator.result - //pick old time as aggregated time + // Picks old time as aggregated time assert(result.time == olderTime) - // do average + // Does average val check = result.value.asInstanceOf[Histogram] assert(check.mean - expect.mean < 0.01) assert(check.stddev - expect.stddev < 0.01) - assert(check.median - expect.median< 0.01) + assert(check.median - expect.median < 0.01) assert(check.p95 - expect.p95 < 0.01) assert(check.p99 - expect.p99 < 0.01) assert(check.p999 - expect.p999 < 0.01) @@ -98,13 +99,13 @@ class ProcessorAggregatorSpec extends FlatSpec with Matchers { val result = aggregator.result - //pick old time + // Picks old time assert(result.time == olderTime) - // do summing + // Does summing val check = result.value.asInstanceOf[Meter] - assert(check.count == expect.count) + assert(check.count == expect.count) assert(check.m1 - expect.m1 < 0.01) assert(check.meanRate - expect.meanRate < 0.01) assert(check.rateUnit == expect.rateUnit) @@ -124,25 +125,34 @@ class ProcessorAggregatorSpec extends FlatSpec with Matchers { } "ProcessorAggregator" should "aggregate on different read options" in { - val hours = 2 // maintain 2 hours history - val seconds = 2 // maintain 2 seconds recent data - val taskCount = 5 // for each processor - val metricCount = 100 // for each task, have metricCount metrics - val range = new HistoryMetricsConfig(hours, hours / 2 * 3600 * 1000, seconds, seconds / 2 * 1000) + val hours = 2 // Maintains 2 hours history + val seconds = 2 // Maintains 2 seconds recent data + val taskCount = 5 // For each processor + val metricCount = 100 // For each task, have metricCount metrics + val range = new HistoryMetricsConfig(hours, hours / 2 * 3600 * 1000, + seconds, seconds / 2 * 1000) val aggregator = new ProcessorAggregator(range) def count(value: Int): Int = value - def inputs(timeRange: Long) = { - (0 until taskCount).map(TaskId(processorId = 0, _)).flatMap(histogram(_, "receiveLatency", timeRange, metricCount)).toList ++ - (0 until taskCount).map(TaskId(processorId = 0, _)).flatMap(histogram(_, "processTime", timeRange, metricCount)).toList ++ - (0 until taskCount).map(TaskId(processorId = 1, _)).flatMap(histogram(_, "receiveLatency", timeRange, metricCount)).toList ++ - (0 until taskCount).map(TaskId(processorId = 1, _)).flatMap(histogram(_, "processTime", timeRange, metricCount)).toList ++ - (0 until taskCount).map(TaskId(processorId = 0, _)).flatMap(meter(_, "sendThroughput", timeRange, metricCount)).toList ++ - (0 until taskCount).map(TaskId(processorId = 0, _)).flatMap(meter(_, "receiveThroughput", timeRange, metricCount)).toList ++ - (0 until taskCount).map(TaskId(processorId = 1, _)).flatMap(meter(_, "sendThroughput", timeRange, metricCount)).toList ++ - (0 until taskCount).map(TaskId(processorId = 1, _)).flatMap(meter(_, "receiveThroughput", timeRange, metricCount)).toList + def inputs(timeRange: Long): List[HistoryMetricsItem] = { + (0 until taskCount).map(TaskId(processorId = 0, _)) + .flatMap(histogram(_, "receiveLatency", timeRange, metricCount)).toList ++ + (0 until taskCount).map(TaskId(processorId = 0, _)) + .flatMap(histogram(_, "processTime", timeRange, metricCount)).toList ++ + (0 until taskCount).map(TaskId(processorId = 1, _)) + .flatMap(histogram(_, "receiveLatency", timeRange, metricCount)).toList ++ + (0 until taskCount).map(TaskId(processorId = 1, _)) + .flatMap(histogram(_, "processTime", timeRange, metricCount)).toList ++ + (0 until taskCount).map(TaskId(processorId = 0, _)) + .flatMap(meter(_, "sendThroughput", timeRange, metricCount)).toList ++ + (0 until taskCount).map(TaskId(processorId = 0, _)) + .flatMap(meter(_, "receiveThroughput", timeRange, metricCount)).toList ++ + (0 until taskCount).map(TaskId(processorId = 1, _)) + .flatMap(meter(_, "sendThroughput", timeRange, metricCount)).toList ++ + (0 until taskCount).map(TaskId(processorId = 1, _)) + .flatMap(meter(_, "receiveThroughput", timeRange, metricCount)).toList } def check(list: List[HistoryMetricsItem], countMap: Map[String, Int]): Boolean = { @@ -150,14 +160,15 @@ class ProcessorAggregatorSpec extends FlatSpec with Matchers { nameCount sameElements countMap } - // aggregate on processor and meterNames, + // Aggregates on processor and meterNames, val input = inputs(Long.MaxValue) - val readLatest = aggregator.aggregate(ReadOption.ReadLatest, input.iterator, now = Long.MaxValue) - assert(readLatest.size == 8) //2 processor * 4 metrics type + val readLatest = aggregator.aggregate(ReadOption.ReadLatest, + input.iterator, now = Long.MaxValue) + assert(readLatest.size == 8) // 2 processor * 4 metrics type assert(check(readLatest, Map( "app0.processor0:receiveLatency" -> count(1), "app0.processor0:processTime" -> count(1), - "app0.processor0:sendThroughput"-> count(1), + "app0.processor0:sendThroughput" -> count(1), "app0.processor0:receiveThroughput" -> count(1), "app0.processor1:receiveLatency" -> count(1), "app0.processor1:processTime" -> count(1), @@ -165,13 +176,14 @@ class ProcessorAggregatorSpec extends FlatSpec with Matchers { "app0.processor1:receiveThroughput" -> count(1) ))) - // aggregate on processor and meterNames and time range, - val readRecent = aggregator.aggregate(ReadOption.ReadRecent, inputs(seconds * 1000).iterator, now = seconds * 1000) - assert(readRecent.size == 16) //2 processor * 4 metrics type * 2 time range + // Aggregates on processor and meterNames and time range, + val readRecent = aggregator.aggregate(ReadOption.ReadRecent, + inputs(seconds * 1000).iterator, now = seconds * 1000) + assert(readRecent.size == 16) // 2 processor * 4 metrics type * 2 time range assert(check(readRecent, Map( "app0.processor0:receiveLatency" -> count(2), "app0.processor0:processTime" -> count(2), - "app0.processor0:sendThroughput"-> count(2), + "app0.processor0:sendThroughput" -> count(2), "app0.processor0:receiveThroughput" -> count(2), "app0.processor1:receiveLatency" -> count(2), "app0.processor1:processTime" -> count(2), @@ -179,13 +191,14 @@ class ProcessorAggregatorSpec extends FlatSpec with Matchers { "app0.processor1:receiveThroughput" -> count(2) ))) - // aggregate on processor and meterNames and time range, - val readHistory = aggregator.aggregate(ReadOption.ReadHistory, inputs(hours * 3600 * 1000).iterator, now = hours * 3600 * 1000) - assert(readHistory.size == 16) //2 processor * 4 metrics type * 2 time ranges + // Aggregates on processor and meterNames and time range, + val readHistory = aggregator.aggregate(ReadOption.ReadHistory, + inputs(hours * 3600 * 1000).iterator, now = hours * 3600 * 1000) + assert(readHistory.size == 16) // 2 processor * 4 metrics type * 2 time ranges assert(check(readHistory, Map( "app0.processor0:receiveLatency" -> count(2), "app0.processor0:processTime" -> count(2), - "app0.processor0:sendThroughput"-> count(2), + "app0.processor0:sendThroughput" -> count(2), "app0.processor0:receiveThroughput" -> count(2), "app0.processor1:receiveLatency" -> count(2), "app0.processor1:processTime" -> count(2), @@ -194,9 +207,11 @@ class ProcessorAggregatorSpec extends FlatSpec with Matchers { ))) } - private def histogram(taskId: TaskId, metricName: String = "latency", timeRange: Long = Long.MaxValue, repeat: Int = 1): List[HistoryMetricsItem] = { + private def histogram( + taskId: TaskId, metricName: String = "latency", timeRange: Long = Long.MaxValue, + repeat: Int = 1): List[HistoryMetricsItem] = { val random = new Random() - (0 until repeat).map {_ => + (0 until repeat).map { _ => new HistoryMetricsItem(Math.abs(random.nextLong() % timeRange), new Histogram(s"app0.processor${taskId.processorId}.task${taskId.index}:$metricName", Math.abs(random.nextDouble()), @@ -209,7 +224,8 @@ class ProcessorAggregatorSpec extends FlatSpec with Matchers { }.toList } - private def meter(taskId: TaskId, metricName: String, timeRange: Long, repeat: Int): List[HistoryMetricsItem] = { + private def meter(taskId: TaskId, metricName: String, timeRange: Long, repeat: Int) + : List[HistoryMetricsItem] = { val random = new Random() (0 until repeat).map { _ => new HistoryMetricsItem(Math.abs(random.nextLong() % timeRange), @@ -225,19 +241,20 @@ class ProcessorAggregatorSpec extends FlatSpec with Matchers { "ProcessorAggregator" should "handle smoothly for unsupported metric type and " + "error formatted metric name" in { val invalid = List( - // unsupported metric type + // Unsupported metric type HistoryMetricsItem(0, new Gauge("app0.processor0.task0:gauge", 100)), - //wrong format: should be app0.processor0.task0:throughput + // Wrong format: should be app0.processor0.task0:throughput HistoryMetricsItem(0, new Meter("app0.processor0.task0/throughput", 100, 0, 0, "")) ) val valid = histogram(TaskId(0, 0), repeat = 10) val aggregator = new ProcessorAggregator(new HistoryMetricsConfig(0, 0, 0, 0)) - val result = aggregator.aggregate(ReadOption.ReadLatest, (valid ++ invalid).toIterator, now = Long.MaxValue) + val result = aggregator.aggregate(ReadOption.ReadLatest, (valid ++ invalid).toIterator, + now = Long.MaxValue) - // for one taskId, will only use one data point. + // For one taskId, will only use one data point. assert(result.size == 1) assert(result.head.value.name == "app0.processor0:latency") } diff --git a/streaming/src/test/scala/io/gearpump/streaming/metrics/TaskFilterAggregatorSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/metrics/TaskFilterAggregatorSpec.scala index 7924acb7d..f654eb96b 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/metrics/TaskFilterAggregatorSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/metrics/TaskFilterAggregatorSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,26 +18,27 @@ package io.gearpump.streaming.metrics +import scala.util.Random + +import org.scalatest.{FlatSpec, Matchers} + import io.gearpump.cluster.MasterToClient.HistoryMetricsItem -import io.gearpump.metrics.Metrics.{Meter, Gauge, Histogram} +import io.gearpump.metrics.Metrics.{Histogram, Meter} import io.gearpump.streaming.metrics.TaskFilterAggregator.Options -import io.gearpump.streaming.task.{StartTime, TaskId} -import org.scalatest.{Matchers, FlatSpec} - -import scala.util.Random +import io.gearpump.streaming.task.TaskId -class TaskFilterAggregatorSpec extends FlatSpec with Matchers { +class TaskFilterAggregatorSpec extends FlatSpec with Matchers { def metric(taskId: TaskId): HistoryMetricsItem = { val random = new Random() new HistoryMetricsItem(Math.abs(random.nextLong()), new Histogram(s"app0.processor${taskId.processorId}.task${taskId.index}:latency", - 0, 0,0,0,0,0)) + 0, 0, 0, 0, 0, 0)) } it should "filter data on processor range, task range combination" in { - val inputs = (0 until 10).flatMap{processor => - (0 until 10).map{ task => + val inputs = (0 until 10).flatMap { processor => + (0 until 10).map { task => metric(TaskId(processor, task)) } }.toList @@ -45,17 +46,17 @@ class TaskFilterAggregatorSpec extends FlatSpec with Matchers { val globalLimit = 10 val aggregator = new TaskFilterAggregator(globalLimit) - // limit not met, return all matches in this matrix + // Limit not met, return all matches in this matrix var options = new Options(limit = 20, startTask = 3, endTask = 6, startProcessor = 3, endProcessor = 6) assert(aggregator.aggregate(options, inputs.iterator).size == 9) - // user limit reached + // User limit reached options = new Options(limit = 3, startTask = 3, endTask = 5, startProcessor = 3, endProcessor = 5) assert(aggregator.aggregate(options, inputs.iterator).size == 3) - // global limit reached + // Global limit reached options = new Options(limit = 20, startTask = 3, endTask = 8, startProcessor = 3, endProcessor = 8) assert(aggregator.aggregate(options, inputs.iterator).size == globalLimit) @@ -67,8 +68,8 @@ class TaskFilterAggregatorSpec extends FlatSpec with Matchers { } it should "skip wrong format metrics" in { - val invalid = List{ - //wrong format: should be app0.processor0.task0:throughput + val invalid = List { + // Wrong format: should be app0.processor0.task0:throughput HistoryMetricsItem(0, new Meter("app0.processor0.task0/throughput", 100, 0, 0, "")) } val options = Options.acceptAll diff --git a/streaming/src/test/scala/io/gearpump/streaming/source/DefaultTimeStampFilterSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/source/DefaultTimeStampFilterSpec.scala index 10bc48e4d..752a8f462 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/source/DefaultTimeStampFilterSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/source/DefaultTimeStampFilterSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,10 +18,11 @@ package io.gearpump.streaming.source -import io.gearpump.{TimeStamp, Message} import org.scalacheck.Gen -import org.scalatest.{Matchers, PropSpec} import org.scalatest.prop.PropertyChecks +import org.scalatest.{Matchers, PropSpec} + +import io.gearpump.{Message, TimeStamp} class DefaultTimeStampFilterSpec extends PropSpec with PropertyChecks with Matchers { property("DefaultTimeStampFilter should filter message against give timestamp") { diff --git a/streaming/src/test/scala/io/gearpump/streaming/state/impl/CheckpointManagerSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/state/impl/CheckpointManagerSpec.scala index 483200f4d..ea670f608 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/state/impl/CheckpointManagerSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/state/impl/CheckpointManagerSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,14 +18,15 @@ package io.gearpump.streaming.state.impl -import io.gearpump.TimeStamp -import io.gearpump.streaming.transaction.api.CheckpointStore -import org.mockito.{Matchers => MockitoMatchers} import org.mockito.Mockito._ +import org.mockito.{Matchers => MockitoMatchers} import org.scalacheck.Gen import org.scalatest.mock.MockitoSugar -import org.scalatest.{Matchers, PropSpec} import org.scalatest.prop.PropertyChecks +import org.scalatest.{Matchers, PropSpec} + +import io.gearpump.TimeStamp +import io.gearpump.streaming.transaction.api.CheckpointStore class CheckpointManagerSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar { @@ -56,7 +57,7 @@ class CheckpointManagerSpec extends PropSpec with PropertyChecks with Matchers w } } - property("CheckpointManager should close CheckpointStore") { + property("CheckpointManager should close CheckpointStore") { forAll(checkpointIntervalGen) { (checkpointInterval: Long) => val checkpointStore = mock[CheckpointStore] @@ -85,5 +86,4 @@ class CheckpointManagerSpec extends PropSpec with PropertyChecks with Matchers w checkpointManager.getCheckpointTime shouldBe empty } } - } diff --git a/streaming/src/test/scala/io/gearpump/streaming/state/impl/InMemoryCheckpointStoreSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/state/impl/InMemoryCheckpointStoreSpec.scala index 86996abb4..4bbad3b9d 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/state/impl/InMemoryCheckpointStoreSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/state/impl/InMemoryCheckpointStoreSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/streaming/src/test/scala/io/gearpump/streaming/state/impl/NonWindowStateSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/state/impl/NonWindowStateSpec.scala index e420a8659..b20b666f3 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/state/impl/NonWindowStateSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/state/impl/NonWindowStateSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,15 +18,16 @@ package io.gearpump.streaming.state.impl -import io.gearpump.TimeStamp -import io.gearpump.streaming.state.api.{Monoid, Serializer} +import scala.util.Success + import org.mockito.Mockito._ import org.scalacheck.Gen import org.scalatest.mock.MockitoSugar import org.scalatest.prop.PropertyChecks import org.scalatest.{Matchers, PropSpec} -import scala.util.Success +import io.gearpump.TimeStamp +import io.gearpump.streaming.state.api.{Monoid, Serializer} class NonWindowStateSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar { @@ -89,7 +90,6 @@ class NonWindowStateSpec extends PropSpec with PropertyChecks with Matchers with state.left shouldBe plus state.right shouldBe zero state.get shouldBe Some(plus) - } } @@ -129,5 +129,4 @@ class NonWindowStateSpec extends PropSpec with PropertyChecks with Matchers with state.get shouldBe Some(plus) } } - } diff --git a/streaming/src/test/scala/io/gearpump/streaming/state/impl/WindowSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/state/impl/WindowSpec.scala index f0489a33a..e12ec9bfa 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/state/impl/WindowSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/state/impl/WindowSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,11 +18,12 @@ package io.gearpump.streaming.state.impl -import io.gearpump.TimeStamp import org.scalacheck.Gen import org.scalatest.mock.MockitoSugar -import org.scalatest.{Matchers, PropSpec} import org.scalatest.prop.PropertyChecks +import org.scalatest.{Matchers, PropSpec} + +import io.gearpump.TimeStamp class WindowSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar { @@ -43,10 +44,10 @@ class WindowSpec extends PropSpec with PropertyChecks with Matchers with Mockito forAll(timestampGen, windowSizeGen, windowStepGen) { (timestamp: TimeStamp, windowSize: Long, windowStep: Long) => val window = new Window(windowSize, windowStep) - window.range shouldBe (0L, windowSize) + window.range shouldBe(0L, windowSize) window.slideOneStep() - window.range shouldBe (windowStep, windowSize + windowStep) + window.range shouldBe(windowStep, windowSize + windowStep) window.slideTo(timestamp) val (startTime, endTime) = window.range diff --git a/streaming/src/test/scala/io/gearpump/streaming/state/impl/WindowStateSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/state/impl/WindowStateSpec.scala index dd54d4137..bc79ff63a 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/state/impl/WindowStateSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/state/impl/WindowStateSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,17 +18,18 @@ package io.gearpump.streaming.state.impl -import io.gearpump._ -import io.gearpump.streaming.MockUtil -import io.gearpump.streaming.state.api.{Serializer, Group} +import scala.collection.immutable.TreeMap +import scala.util.Success + import org.mockito.Mockito._ import org.scalacheck.Gen import org.scalatest.mock.MockitoSugar import org.scalatest.prop.PropertyChecks import org.scalatest.{Matchers, PropSpec} -import scala.collection.immutable.TreeMap -import scala.util.Success +import io.gearpump._ +import io.gearpump.streaming.MockUtil +import io.gearpump.streaming.state.api.{Group, Serializer} class WindowStateSpec extends PropSpec with PropertyChecks with Matchers with MockitoSugar { @@ -91,7 +92,7 @@ class WindowStateSpec extends PropSpec with PropertyChecks with Matchers with Mo state.right shouldBe zero state.get shouldBe Some(zero) - val start = checkpointTime - 1 + val start = checkpointTime - 1 val end = checkpointTime + 1 val size = end - start val step = 1L @@ -130,11 +131,10 @@ class WindowStateSpec extends PropSpec with PropertyChecks with Matchers with Mo val right = mock[AnyRef] val plus = mock[AnyRef] - when(group.zero).thenReturn(zero, zero) val state = new WindowState[AnyRef](group, serializer, taskContext, window) - val start = checkpointTime - 1 + val start = checkpointTime - 1 val end = checkpointTime + 1 val size = end - start val step = 1L @@ -147,8 +147,8 @@ class WindowStateSpec extends PropSpec with PropertyChecks with Matchers with Mo when(group.plus(left, zero)).thenReturn(left, Nil: _*) when(taskContext.upstreamMinClock).thenReturn(0L) - // time < checkpointTime - // update left in current window + // Time < checkpointTime + // Update left in current window state.setNextCheckpointTime(checkpointTime) state.update(start, left) @@ -166,8 +166,8 @@ class WindowStateSpec extends PropSpec with PropertyChecks with Matchers with Mo when(group.plus(left, right)).thenReturn(plus, Nil: _*) when(taskContext.upstreamMinClock).thenReturn(0L) - // time >= checkpointTime - // update right in current window + // Time >= checkpointTime + // Update right in current window state.setNextCheckpointTime(checkpointTime) state.update(checkpointTime, right) @@ -176,10 +176,9 @@ class WindowStateSpec extends PropSpec with PropertyChecks with Matchers with Mo state.right shouldBe right state.get shouldBe Some(plus) state.getIntervalStates(start, end) shouldBe - TreeMap(Interval(start, start + step) -> left, Interval(start + step, end) -> right) - + TreeMap(Interval(start, start + step) -> left, Interval(start + step, end) -> right) - // slide window + // Slides window forward when(window.range).thenReturn((start, end), (start + step, end + step)) when(window.shouldSlide).thenReturn(true) when(taskContext.upstreamMinClock).thenReturn(checkpointTime) @@ -197,10 +196,10 @@ class WindowStateSpec extends PropSpec with PropertyChecks with Matchers with Mo state.right shouldBe plus state.get shouldBe Some(plus) state.getIntervalStates(start, end + step) shouldBe - TreeMap( - Interval(start, start + step) -> left, - Interval(start + step, end) -> right, - Interval(end, end + step) -> right) + TreeMap( + Interval(start, start + step) -> left, + Interval(start + step, end) -> right, + Interval(end, end + step) -> right) } } @@ -232,7 +231,8 @@ class WindowStateSpec extends PropSpec with PropertyChecks with Matchers with Mo timestamp / windowStep * windowStep should (be <= interval.startTime) (timestamp - windowSize) / windowStep * windowStep should (be <= interval.startTime) (timestamp / windowStep + 1) * windowStep should (be >= interval.endTime) - ((timestamp - windowSize) / windowStep + 1) * windowStep + windowSize should (be >= interval.endTime) + ((timestamp - windowSize) / windowStep + 1) * windowStep + windowSize should + (be >= interval.endTime) checkpointTime should (be <= interval.startTime or be >= interval.endTime) } } diff --git a/streaming/src/test/scala/io/gearpump/streaming/storage/InMemoryAppStoreOnMasterSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/storage/InMemoryAppStoreOnMasterSpec.scala index 509946603..60baa742e 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/storage/InMemoryAppStoreOnMasterSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/storage/InMemoryAppStoreOnMasterSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,17 +17,16 @@ */ package io.gearpump.streaming.storage -import io.gearpump.streaming.StreamingTestUtil -import io.gearpump.cluster.{MasterHarness, MiniCluster, TestUtil} -import io.gearpump.streaming.StreamingTestUtil -import io.gearpump.util.Constants -import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpec} - import scala.concurrent.Await import scala.concurrent.duration._ -import scala.language.postfixOps -class InMemoryAppStoreOnMasterSpec extends WordSpec with Matchers with BeforeAndAfterAll{ +import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpec} + +import io.gearpump.cluster.{MasterHarness, MiniCluster} +import io.gearpump.streaming.StreamingTestUtil +import io.gearpump.util.Constants + +class InMemoryAppStoreOnMasterSpec extends WordSpec with Matchers with BeforeAndAfterAll { implicit val timeout = Constants.FUTURE_TIMEOUT implicit val dispatcher = MasterHarness.cachedPool @@ -45,14 +44,18 @@ class InMemoryAppStoreOnMasterSpec extends WordSpec with Matchers with BeforeAnd store.put("Int_type", 1024) store.put("Tuple2_type", ("element1", 1024)) - val future1 = store.get("String_type").map { value => value.asInstanceOf[String] should be("this is a string")} - val future2 = store.get("Int_type").map { value => value.asInstanceOf[Int] should be(1024)} - val future3 = store.get("Tuple2_type").map { value => value.asInstanceOf[(String, Int)] should be(("element1", 1024))} - val future4 = store.get("key").map { value => value.asInstanceOf[Object] should be(null)} - Await.result(future1, 15 seconds) - Await.result(future2, 15 seconds) - Await.result(future3, 15 seconds) - Await.result(future4, 15 seconds) + val future1 = store.get("String_type").map { value => + value.asInstanceOf[String] should be("this is a string") + } + val future2 = store.get("Int_type").map { value => value.asInstanceOf[Int] should be(1024) } + val future3 = store.get("Tuple2_type").map { value => + value.asInstanceOf[(String, Int)] should be(("element1", 1024)) + } + val future4 = store.get("key").map { value => value.asInstanceOf[Object] should be(null) } + Await.result(future1, 15.seconds) + Await.result(future2, 15.seconds) + Await.result(future3, 15.seconds) + Await.result(future4, 15.seconds) miniCluster.shutDown } } diff --git a/streaming/src/test/scala/io/gearpump/streaming/task/SubscriberSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/task/SubscriberSpec.scala index ffc6df17a..ffb343eae 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/task/SubscriberSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/task/SubscriberSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,29 +17,32 @@ */ package io.gearpump.streaming.task +import org.scalatest.{FlatSpec, Matchers} + import io.gearpump.partitioner.{HashPartitioner, Partitioner} import io.gearpump.streaming.task.SubscriberSpec.TestTask import io.gearpump.streaming.{DAG, ProcessorDescription} import io.gearpump.util.Graph import io.gearpump.util.Graph._ -import org.scalatest.{FlatSpec, Matchers} -class SubscriberSpec extends FlatSpec with Matchers { +class SubscriberSpec extends FlatSpec with Matchers { "Subscriber.of" should "return all subscriber for a processor" in { val sourceProcessorId = 0 - val task1 = ProcessorDescription(id = sourceProcessorId, taskClass = classOf[TestTask].getName, parallelism = 1) + val task1 = ProcessorDescription(id = sourceProcessorId, taskClass = + classOf[TestTask].getName, parallelism = 1) val task2 = ProcessorDescription(id = 1, taskClass = classOf[TestTask].getName, parallelism = 1) val task3 = ProcessorDescription(id = 2, taskClass = classOf[TestTask].getName, parallelism = 1) val partitioner = Partitioner[HashPartitioner] - val dag = DAG(Graph(task1 ~ partitioner ~> task2, task1 ~ partitioner ~> task3, task2 ~ partitioner ~> task3)) - + val dag = DAG(Graph(task1 ~ partitioner ~> task2, task1 ~ partitioner ~> task3, + task2 ~ partitioner ~> task3)) val subscribers = Subscriber.of(sourceProcessorId, dag) assert(subscribers.size == 2) assert(subscribers.toSet == - Set(Subscriber(1, partitioner, task2.parallelism, task2.life), Subscriber(2, partitioner, task3.parallelism, task3.life))) + Set(Subscriber(1, partitioner, task2.parallelism, task2.life), Subscriber(2, partitioner, + task3.parallelism, task3.life))) } } diff --git a/streaming/src/test/scala/io/gearpump/streaming/task/SubscriptionSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/task/SubscriptionSpec.scala index ecad47b3d..57f4b8c58 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/task/SubscriptionSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/task/SubscriptionSpec.scala @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,16 +20,15 @@ package io.gearpump.streaming.task import java.util.Random +import org.mockito.Mockito._ +import org.scalatest.mock.MockitoSugar +import org.scalatest.{FlatSpec, Matchers} + import io.gearpump.Message import io.gearpump.cluster.UserConfig -import io.gearpump.partitioner.{Partitioner, HashPartitioner} -import io.gearpump.streaming.{LifeTime, ProcessorDescription} +import io.gearpump.partitioner.{HashPartitioner, Partitioner} import io.gearpump.streaming.task.SubscriptionSpec.NextTask -import org.scalatest.{FlatSpec} - -import org.mockito.Mockito._ -import org.scalatest.{Matchers} -import org.scalatest.mock.MockitoSugar +import io.gearpump.streaming.{LifeTime, ProcessorDescription} class SubscriptionSpec extends FlatSpec with Matchers with MockitoSugar { val appId = 0 @@ -41,16 +40,18 @@ class SubscriptionSpec extends FlatSpec with Matchers with MockitoSugar { val partitioner = Partitioner[HashPartitioner] val parallism = 2 - val downstreamProcessor = ProcessorDescription(downstreamProcessorId, classOf[NextTask].getName, parallism) - val subscriber = Subscriber(downstreamProcessorId, partitioner, downstreamProcessor.parallelism, downstreamProcessor.life) + val downstreamProcessor = ProcessorDescription(downstreamProcessorId, classOf[NextTask].getName, + parallism) + val subscriber = Subscriber(downstreamProcessorId, partitioner, downstreamProcessor.parallelism, + downstreamProcessor.life) private def prepare: (Subscription, ExpressTransport) = { val transport = mock[ExpressTransport] val subscription = new Subscription(appId, executorId, taskId, subscriber, session, transport) - subscription.start + subscription.start() val expectedAckRequest = InitialAckRequest(taskId, session) - verify(transport, times(1)).transport(expectedAckRequest, TaskId(1,0), TaskId(1, 1)) + verify(transport, times(1)).transport(expectedAckRequest, TaskId(1, 0), TaskId(1, 1)) (subscription, transport) } @@ -67,41 +68,40 @@ class SubscriptionSpec extends FlatSpec with Matchers with MockitoSugar { val msg1 = new Message("1", timestamp = 70) subscription.sendMessage(msg1) - verify(transport, times(1)).transport(msg1, TaskId(1,1)) + verify(transport, times(1)).transport(msg1, TaskId(1, 1)) assert(subscription.minClock == 70) val msg2 = new Message("0", timestamp = 50) subscription.sendMessage(msg2) - verify(transport, times(1)).transport(msg2, TaskId(1,0)) + verify(transport, times(1)).transport(msg2, TaskId(1, 0)) // minClock has been set to smaller one assert(subscription.minClock == 50) val initialMinClock = subscription.minClock - //ack initial AckRequest(0) + // Acks initial AckRequest(0) subscription.receiveAck(Ack(TaskId(1, 1), 0, 0, session)) subscription.receiveAck(Ack(TaskId(1, 0), 0, 0, session)) - //send 100 messages - 100 until 200 foreach {clock => + // Sends 100 messages + 100 until 200 foreach { clock => subscription.sendMessage(Message("1", clock)) subscription.sendMessage(Message("2", clock)) } - // ack not received, minClock no change + // Ack not received, minClock no change assert(subscription.minClock == initialMinClock) subscription.receiveAck(Ack(TaskId(1, 1), 100, 100, session)) subscription.receiveAck(Ack(TaskId(1, 0), 100, 100, session)) - // ack received, minClock changed + // Ack received, minClock changed assert(subscription.minClock > initialMinClock) - - // we expect to receive two ackRequest for two downstream tasks + // Expects to receive two ackRequest for two downstream tasks val ackRequestForTask0 = AckRequest(taskId, 200, session) - verify(transport, times(1)).transport(ackRequestForTask0, TaskId(1,0)) + verify(transport, times(1)).transport(ackRequestForTask0, TaskId(1, 0)) val ackRequestForTask1 = AckRequest(taskId, 200, session) verify(transport, times(1)).transport(ackRequestForTask1, TaskId(1, 1)) @@ -109,13 +109,12 @@ class SubscriptionSpec extends FlatSpec with Matchers with MockitoSugar { it should "disallow more message sending if there is no ack back" in { val (subscription, transport) = prepare - //send 100 messages - 0 until (Subscription.MAX_PENDING_MESSAGE_COUNT * 2 + 1) foreach {clock => + // send 100 messages + 0 until (Subscription.MAX_PENDING_MESSAGE_COUNT * 2 + 1) foreach { clock => subscription.sendMessage(Message(randomMessage, clock)) } assert(subscription.allowSendingMoreMessages() == false) - } it should "report minClock as Long.MaxValue when there is no pending message" in { @@ -128,19 +127,16 @@ class SubscriptionSpec extends FlatSpec with Matchers with MockitoSugar { } private def randomMessage: String = new Random().nextInt.toString - } object SubscriptionSpec { + class NextTask(taskContext: TaskContext, conf: UserConfig) extends Task(taskContext, conf) { - class NextTask(taskContext : TaskContext, conf: UserConfig) extends Task(taskContext, conf) { - import taskContext.{output, self} - - override def onStart(startTime : StartTime) : Unit = { + override def onStart(startTime: StartTime): Unit = { } - override def onNext(msg : Message) : Unit = { + override def onNext(msg: Message): Unit = { } } } \ No newline at end of file diff --git a/streaming/src/test/scala/io/gearpump/streaming/task/TaskActorSpec.scala b/streaming/src/test/scala/io/gearpump/streaming/task/TaskActorSpec.scala index 275cdb046..a48f887ef 100644 --- a/streaming/src/test/scala/io/gearpump/streaming/task/TaskActorSpec.scala +++ b/streaming/src/test/scala/io/gearpump/streaming/task/TaskActorSpec.scala @@ -1,44 +1,46 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.gearpump.streaming.task import akka.actor.{ExtendedActorSystem, Props} import akka.testkit._ -import com.typesafe.config.ConfigFactory +import com.typesafe.config.{Config, ConfigFactory} +import org.mockito.Mockito.{mock, times, verify, when} +import org.scalatest.{BeforeAndAfterEach, Matchers, WordSpec} + import io.gearpump.Message import io.gearpump.cluster.{MasterHarness, TestUtil, UserConfig} import io.gearpump.partitioner.{HashPartitioner, Partitioner} import io.gearpump.serializer.{FastKryoSerializer, SerializationFramework} -import io.gearpump.streaming.AppMasterToExecutor.{StartTask, TaskRegistered, ChangeTask, MsgLostException, TaskChanged} +import io.gearpump.streaming.AppMasterToExecutor.{ChangeTask, MsgLostException, StartTask, TaskChanged, TaskRegistered} import io.gearpump.streaming.task.TaskActorSpec.TestTask import io.gearpump.streaming.{DAG, LifeTime, ProcessorDescription} import io.gearpump.util.Graph._ import io.gearpump.util.{Graph, Util} -import org.mockito.Mockito.{mock, times, verify, when} -import org.scalatest.{BeforeAndAfterEach, Matchers, WordSpec} - class TaskActorSpec extends WordSpec with Matchers with BeforeAndAfterEach with MasterHarness { - override def config = ConfigFactory.parseString( - """ akka.loggers = ["akka.testkit.TestEventListener"] - | akka.test.filter-leeway = 20000 - """.stripMargin). - withFallback(TestUtil.DEFAULT_CONFIG) + protected override def config: Config = { + ConfigFactory.parseString( + """ akka.loggers = ["akka.testkit.TestEventListener"] + | akka.test.filter-leeway = 20000 + """.stripMargin). + withFallback(TestUtil.DEFAULT_CONFIG) + } val appId = 0 val task1 = ProcessorDescription(id = 0, taskClass = classOf[TestTask].getName, parallelism = 1) @@ -54,7 +56,7 @@ class TaskActorSpec extends WordSpec with Matchers with BeforeAndAfterEach with var mockSerializerPool: SerializationFramework = null - override def beforeEach() = { + override def beforeEach(): Unit = { startActorSystem() mockMaster = TestProbe()(getActorSystem) @@ -71,8 +73,10 @@ class TaskActorSpec extends WordSpec with Matchers with BeforeAndAfterEach with "TaskActor" should { "register itself to AppMaster when started" in { val mockTask = mock(classOf[TaskWrapper]) - val testActor = TestActorRef[TaskActor](Props(new TaskActor(taskId1, taskContext1, UserConfig.empty, mockTask, mockSerializerPool)))(getActorSystem) - testActor ! TaskRegistered(taskId1, 0, Util.randInt) + val testActor = TestActorRef[TaskActor](Props( + new TaskActor(taskId1, taskContext1, UserConfig.empty, + mockTask, mockSerializerPool)))(getActorSystem) + testActor ! TaskRegistered(taskId1, 0, Util.randInt()) testActor ! StartTask(taskId1) implicit val system = getActorSystem @@ -84,8 +88,9 @@ class TaskActorSpec extends WordSpec with Matchers with BeforeAndAfterEach with "respond to ChangeTask" in { val mockTask = mock(classOf[TaskWrapper]) - val testActor = TestActorRef[TaskActor](Props(new TaskActor(taskId1, taskContext1, UserConfig.empty, mockTask, mockSerializerPool)))(getActorSystem) - testActor ! TaskRegistered(taskId1, 0, Util.randInt) + val testActor = TestActorRef[TaskActor](Props(new TaskActor(taskId1, taskContext1, + UserConfig.empty, mockTask, mockSerializerPool)))(getActorSystem) + testActor ! TaskRegistered(taskId1, 0, Util.randInt()) testActor ! StartTask(taskId1) mockMaster.expectMsgType[GetUpstreamMinClock] @@ -97,8 +102,9 @@ class TaskActorSpec extends WordSpec with Matchers with BeforeAndAfterEach with val mockTask = mock(classOf[TaskWrapper]) val msg = Message("test") - val testActor = TestActorRef[TaskActor](Props(new TaskActor(taskId1, taskContext1, UserConfig.empty, mockTask, mockSerializerPool)))(getActorSystem) - testActor.tell(TaskRegistered(taskId1, 0, Util.randInt), mockMaster.ref) + val testActor = TestActorRef[TaskActor](Props(new TaskActor(taskId1, taskContext1, + UserConfig.empty, mockTask, mockSerializerPool)))(getActorSystem) + testActor.tell(TaskRegistered(taskId1, 0, Util.randInt()), mockMaster.ref) testActor.tell(StartTask(taskId1), mockMaster.ref) testActor.tell(msg, testActor) @@ -107,7 +113,7 @@ class TaskActorSpec extends WordSpec with Matchers with BeforeAndAfterEach with } } - override def afterEach() = { + override def afterEach(): Unit = { shutdownActorSystem() } } diff --git a/unmanagedlibs/README.md b/unmanagedlibs/README.md index 23a5895e9..7c51d2c36 100644 --- a/unmanagedlibs/README.md +++ b/unmanagedlibs/README.md @@ -1,5 +1,3 @@ -This folder contains the unmanaged dependency files that couldn't be achieved from public repositories. +This folder contains the unmanaged dependency libraries. We will copy the jars under directory {scala-version}/ to out/target/pack/lib/ -Related issue: https://github.com/gearpump/gearpump/issues/1816 - -We built a new akka-actor artifact to replace the flawed one. \ No newline at end of file +For example, jars under 2.11/ will be copied to output/target/pack/lib when building for scala 2.11. \ No newline at end of file diff --git a/version.sbt b/version.sbt index e74ced496..7b23e02be 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + version in ThisBuild := "0.8.1-SNAPSHOT" diff --git a/yarnconf/README.md b/yarnconf/README.md index 38fa81853..87b6b3d44 100644 --- a/yarnconf/README.md +++ b/yarnconf/README.md @@ -1,8 +1,10 @@ -### Put YARN configuration files under classpath +This directory contains YARN configurations that you want to use when launching Gearpump over YARN. + +### How to put YARN configuration files under classpath? Before calling "yarnclient launch", make sure you have put all YARN configuration files under classpath. Typically, you can just copy all files under $HADOOP_HOME/etc/hadoop from one of the YARN Cluster machine to "conf/yarnconf" of gearpump. -NOTE: The "conf/yarnconf" is only effecive when you run "yarnclient". When you -run other commands like "gear", "conf/yarnconf" will not be addded into classpath. \ No newline at end of file +NOTE: The "conf/yarnconf" is only effective when you run "yarnclient". When you +run other commands like "gear", "conf/yarnconf" will not be added into classpath. \ No newline at end of file