Bump artifacts to latest release (#179)

Author: robert3005

common/network-common/src/main/java/org/apache/spark/network/crypto/TransportCipher.java

@@ -203,6 +203,11 @@ public long transfered() {
       return transferred;
     }
 
+    @Override
+    public long transferred() {
+      return transferred;
+    }
+
     @Override
     public long transferTo(WritableByteChannel target, long position) throws IOException {
       Preconditions.checkArgument(position == transfered(), "Invalid position.");
@@ -232,7 +237,7 @@ private void encryptMore() throws IOException {
         int copied = byteRawChannel.write(buf.nioBuffer());
         buf.skipBytes(copied);
       } else {
-        region.transferTo(byteRawChannel, region.transfered());
+        region.transferTo(byteRawChannel, region.transferred());
       }
       cos.write(byteRawChannel.getData(), 0, byteRawChannel.length());
       cos.flush();
@@ -241,6 +246,28 @@ private void encryptMore() throws IOException {
         0, byteEncChannel.length());
     }
 
+    @Override
+    public FileRegion retain() {
+      super.retain();
+      return this;
+    }
+
+    @Override
+    public FileRegion retain(int increment) {
+      super.retain(increment);
+      return this;
+    }
+
+    @Override
+    public FileRegion touch() {
+      return this;
+    }
+
+    @Override
+    public FileRegion touch(Object o) {
+      return this;
+    }
+
     @Override
     protected void deallocate() {
       byteRawChannel.reset();

common/network-common/src/main/java/org/apache/spark/network/protocol/MessageWithHeader.java

@@ -95,6 +95,11 @@ public long transfered() {
     return totalBytesTransferred;
   }
 
+  @Override
+  public long transferred() {
+    return totalBytesTransferred;
+  }
+
   /**
    * This code is more complicated than you would think because we might require multiple
    * transferTo invocations in order to transfer a single MessageWithHeader to avoid busy waiting.
@@ -127,6 +132,28 @@ public long transferTo(final WritableByteChannel target, final long position) th
     return writtenHeader + writtenBody;
   }
 
+  @Override
+  public FileRegion touch(Object msg) {
+    return this;
+  }
+
+  @Override
+  public FileRegion retain() {
+    super.retain();
+    return this;
+  }
+
+  @Override
+  public FileRegion retain(int increment) {
+    super.retain(increment);
+    return this;
+  }
+
+  @Override
+  public FileRegion touch() {
+    return this;
+  }
+
   @Override
   protected void deallocate() {
     header.release();

common/network-common/src/main/java/org/apache/spark/network/sasl/SaslEncryption.java

@@ -187,6 +187,11 @@ public long transfered() {
       return transferred;
     }
 
+    @Override
+    public long transferred() {
+      return transferred;
+    }
+
     /**
      * Transfers data from the original message to the channel, encrypting it in the process.
      *
@@ -262,7 +267,7 @@ private void nextChunk() throws IOException {
         int copied = byteChannel.write(buf.nioBuffer());
         buf.skipBytes(copied);
       } else {
-        region.transferTo(byteChannel, region.transfered());
+        region.transferTo(byteChannel, region.transferred());
       }
 
       byte[] encrypted = backend.wrap(byteChannel.getData(), 0, byteChannel.length());
@@ -272,6 +277,28 @@ private void nextChunk() throws IOException {
       this.unencryptedChunkSize = byteChannel.length();
     }
 
+    @Override
+    public FileRegion touch(Object o) {
+      return this;
+    }
+
+    @Override
+    public FileRegion retain() {
+      super.retain();
+      return this;
+    }
+
+    @Override
+    public FileRegion retain(int increment) {
+      super.retain(increment);
+      return this;
+    }
+
+    @Override
+    public FileRegion touch() {
+      return this;
+    }
+
     @Override
     protected void deallocate() {
       if (currentHeader != null) {

common/network-common/src/main/java/org/apache/spark/network/server/TransportServer.java

@@ -140,11 +140,12 @@ public void close() {
       channelFuture.channel().close().awaitUninterruptibly(10, TimeUnit.SECONDS);
       channelFuture = null;
     }
-    if (bootstrap != null && bootstrap.group() != null) {
-      bootstrap.group().shutdownGracefully();
+    if (bootstrap != null && bootstrap.config() != null && bootstrap.config().group() != null) {
+      bootstrap.config().group().shutdownGracefully();
     }
-    if (bootstrap != null && bootstrap.childGroup() != null) {
-      bootstrap.childGroup().shutdownGracefully();
+    if (bootstrap != null && bootstrap.config() != null
+            && bootstrap.config().childGroup() != null) {
+      bootstrap.config().childGroup().shutdownGracefully();
     }
     bootstrap = null;
   }

common/network-common/src/test/java/org/apache/spark/network/ProtocolSuite.java

@@ -56,7 +56,7 @@ private void testServerToClient(Message msg) {
         NettyUtils.createFrameDecoder(), MessageDecoder.INSTANCE);
 
     while (!serverChannel.outboundMessages().isEmpty()) {
-      clientChannel.writeInbound(serverChannel.readOutbound());
+      clientChannel.writeOneInbound(serverChannel.readOutbound());
     }
 
     assertEquals(1, clientChannel.inboundMessages().size());
@@ -72,7 +72,7 @@ private void testClientToServer(Message msg) {
         NettyUtils.createFrameDecoder(), MessageDecoder.INSTANCE);
 
     while (!clientChannel.outboundMessages().isEmpty()) {
-      serverChannel.writeInbound(clientChannel.readOutbound());
+      serverChannel.writeOneInbound(clientChannel.readOutbound());
     }
 
     assertEquals(1, serverChannel.inboundMessages().size());
@@ -116,8 +116,8 @@ public void encode(ChannelHandlerContext ctx, FileRegion in, List<Object> out)
       throws Exception {
 
       ByteArrayWritableChannel channel = new ByteArrayWritableChannel(Ints.checkedCast(in.count()));
-      while (in.transfered() < in.count()) {
-        in.transferTo(channel, in.transfered());
+      while (in.transferred() < in.count()) {
+        in.transferTo(channel, in.transferred());
       }
       out.add(Unpooled.wrappedBuffer(channel.getData()));
     }

common/network-common/src/test/java/org/apache/spark/network/protocol/MessageWithHeaderSuite.java

@@ -134,6 +134,11 @@ public long transfered() {
       return 8 * written;
     }
 
+    @Override
+    public long transferred() {
+      return 8 * written;
+    }
+
     @Override
     public long transferTo(WritableByteChannel target, long position) throws IOException {
       for (int i = 0; i < writesPerCall; i++) {
@@ -148,6 +153,28 @@ public long transferTo(WritableByteChannel target, long position) throws IOExcep
       return 8 * writesPerCall;
     }
 
+    @Override
+    public FileRegion retain() {
+      super.retain();
+      return this;
+    }
+
+    @Override
+    public FileRegion retain(int increment) {
+      super.retain(increment);
+      return this;
+    }
+
+    @Override
+    public FileRegion touch(Object o) {
+      return this;
+    }
+
+    @Override
+    public FileRegion touch() {
+      return this;
+    }
+
     @Override
     protected void deallocate() {
     }

core/src/main/scala/org/apache/spark/storage/DiskStore.scala

@@ -266,6 +266,22 @@ private class ReadableChannelFileRegion(source: ReadableByteChannel, blockSize:
   }
 
   override def deallocate(): Unit = source.close()
+
+  override def transferred(): Long = _transferred
+
+  override def touch(o: scala.Any): FileRegion = this
+
+  override def retain(): FileRegion = {
+    super.retain()
+    this
+  }
+
+  override def retain(increment: Int): FileRegion = {
+    super.retain(increment)
+    this
+  }
+
+  override def touch(): FileRegion = this
 }
 
 private class CountingWritableChannel(sink: WritableByteChannel) extends WritableByteChannel {

dev/deps/spark-deps-hadoop-palantir

@@ -1,13 +1,13 @@
 JavaEWAH-0.3.2.jar
-RoaringBitmap-0.5.11.jar
+RoaringBitmap-0.6.43.jar
 ST4-4.0.4.jar
 activation-1.1.1.jar
 animal-sniffer-annotation-1.0.jar
 antlr-2.7.7.jar
 antlr-runtime-3.4.jar
-antlr4-runtime-4.5.3.jar
+antlr4-runtime-4.7.jar
 aopalliance-1.0.jar
-aopalliance-repackaged-2.4.0-b34.jar
+aopalliance-repackaged-2.5.0-b32.jar
 apache-log4j-extras-1.2.17.jar
 apacheds-i18n-2.0.0-M15.jar
 apacheds-kerberos-codec-2.0.0-M15.jar
@@ -89,12 +89,12 @@ hadoop-yarn-client-2.8.0-palantir3.jar
 hadoop-yarn-common-2.8.0-palantir3.jar
 hadoop-yarn-server-common-2.8.0-palantir3.jar
 hadoop-yarn-server-web-proxy-2.8.0-palantir3.jar
-hk2-api-2.4.0-b34.jar
-hk2-locator-2.4.0-b34.jar
-hk2-utils-2.4.0-b34.jar
+hk2-api-2.5.0-b32.jar
+hk2-locator-2.5.0-b32.jar
+hk2-utils-2.5.0-b32.jar
 htrace-core4-4.0.1-incubating.jar
-httpclient-4.5.2.jar
-httpcore-4.4.4.jar
+httpclient-4.5.3.jar
+httpcore-4.4.6.jar
 ion-java-1.0.1.jar
 ivy-2.4.0.jar
 jackson-annotations-2.6.5.jar
@@ -113,24 +113,24 @@ jackson-module-scala_2.11-2.6.5.jar
 jackson-xc-1.9.13.jar
 janino-3.0.0.jar
 java-xmlbuilder-1.0.jar
-javassist-3.18.1-GA.jar
+javassist-3.20.0-GA.jar
 javax.annotation-api-1.2.jar
 javax.inject-1.jar
-javax.inject-2.4.0-b34.jar
+javax.inject-2.5.0-b32.jar
 javax.servlet-api-3.1.0.jar
 javax.ws.rs-api-2.0.1.jar
 javolution-5.5.1.jar
 jaxb-api-2.2.2.jar
 jcip-annotations-1.0.jar
-jcl-over-slf4j-1.7.16.jar
+jcl-over-slf4j-1.7.25.jar
 jdo-api-3.0.1.jar
-jersey-client-2.22.2.jar
-jersey-common-2.22.2.jar
-jersey-container-servlet-2.22.2.jar
-jersey-container-servlet-core-2.22.2.jar
-jersey-guava-2.22.2.jar
-jersey-media-jaxb-2.22.2.jar
-jersey-server-2.22.2.jar
+jersey-client-2.25.1.jar
+jersey-common-2.25.1.jar
+jersey-container-servlet-2.25.1.jar
+jersey-container-servlet-core-2.25.1.jar
+jersey-guava-2.25.1.jar
+jersey-media-jaxb-2.25.1.jar
+jersey-server-2.25.1.jar
 jets3t-0.9.3.jar
 jetty-6.1.26.jar
 jetty-sslengine-6.1.26.jar
@@ -144,10 +144,10 @@ json4s-ast_2.11-3.2.11.jar
 json4s-core_2.11-3.2.11.jar
 json4s-jackson_2.11-3.2.11.jar
 jsp-api-2.1.jar
-jsr305-1.3.9.jar
+jsr305-3.0.1.jar
 jta-1.1.jar
 jtransforms-2.4.0.jar
-jul-to-slf4j-1.7.16.jar
+jul-to-slf4j-1.7.25.jar
 kryo-shaded-3.0.3.jar
 kubernetes-client-2.2.13.jar
 kubernetes-model-1.0.67.jar
@@ -166,13 +166,13 @@ metrics-json-3.1.2.jar
 metrics-jvm-3.1.2.jar
 minlog-1.3.0.jar
 mx4j-3.0.2.jar
-netty-3.9.9.Final.jar
-netty-all-4.0.43.Final.jar
+netty-3.10.6.Final.jar
+netty-all-4.1.9.Final.jar
 nimbus-jose-jwt-3.9.jar
 objenesis-2.1.jar
-okhttp-2.4.0.jar
-okhttp-3.6.0.jar
-okio-1.11.0.jar
+okhttp-2.7.5.jar
+okhttp-3.7.0.jar
+okio-1.12.0.jar
 opencsv-2.3.jar
 oro-2.0.8.jar
 osgi-resource-locator-1.0.1.jar
@@ -196,8 +196,8 @@ scala-reflect-2.11.8.jar
 scala-xml_2.11-1.0.2.jar
 scalap-2.11.8.jar
 shapeless_2.11-2.3.2.jar
-slf4j-api-1.7.16.jar
-slf4j-log4j12-1.7.16.jar
+slf4j-api-1.7.25.jar
+slf4j-log4j12-1.7.25.jar
 snakeyaml-1.15.jar
 snappy-0.2.jar
 snappy-java-1.1.2.6.jar