diff --git a/pom.xml b/pom.xml index a7dddbd373..f539f9af34 100644 --- a/pom.xml +++ b/pom.xml @@ -898,6 +898,7 @@ apache-rat-plugin + **/*.json CHANGES.txt **/LICENSE* diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java index 8dade76cd3..0a3c14fd69 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java @@ -110,6 +110,9 @@ import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair; import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; import com.google.common.base.Strings; @@ -200,6 +203,16 @@ static boolean setupTezJarsLocalResources(TezConfiguration conf, return usingTezArchive; } + public static ServicePluginsDescriptor createPluginsDescriptorFromJSON(InputStream is) throws IOException { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + if (is != null) { + return objectMapper.readValue(is, ServicePluginsDescriptor.class); + } else { + return ServicePluginsDescriptor.create(false); + } + } + private static boolean addLocalResources(Configuration conf, String[] configUris, Map tezJarResources, Credentials credentials) throws IOException { @@ -834,7 +847,7 @@ public static void addLog4jSystemProperties(String logLevel, } } - static ConfigurationProto createFinalConfProtoForApp(Configuration amConf, + public static ConfigurationProto createFinalConfProtoForApp(Configuration amConf, ServicePluginsDescriptor servicePluginsDescriptor) { assert amConf != null; ConfigurationProto.Builder builder = ConfigurationProto.newBuilder(); diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/NamedEntityDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/NamedEntityDescriptor.java index 86ae26c876..db8527be36 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/NamedEntityDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/NamedEntityDescriptor.java @@ -23,7 +23,14 @@ @SuppressWarnings("unchecked") public class NamedEntityDescriptor> extends EntityDescriptor> { - private final String entityName; + + private String entityName; + + /** + * Public constructor to allow this descriptor to be instantiated by Jackson. + */ + @InterfaceAudience.Private + public NamedEntityDescriptor() {} @InterfaceAudience.Private public NamedEntityDescriptor(String entityName, String className) { diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index c81549ceef..5baa6c1526 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -2332,4 +2332,11 @@ static Set getPropertySet() { @ConfigurationScope(Scope.DAG) @ConfigurationProperty public static final String TEZ_TASK_ATTEMPT_HOOKS = TEZ_TASK_PREFIX + "attempt.hooks"; + + /** + * Comma-separated list of additional hadoop config files to load from CLASSPATH in standalone mode. + */ + @ConfigurationScope(Scope.AM) + @ConfigurationProperty + public static final String TEZ_AM_STANDALONE_CONFS = TEZ_AM_PREFIX + "standalone.confs"; } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java index 379eb0cb1b..9bf5e0503d 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java @@ -49,6 +49,7 @@ public final class TezConstants { TEZ_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT = "security.job.client.protocol.acl"; + public static final String SERVICE_PLUGINS_DESCRIPTOR_JSON = "service_plugins_descriptor.json"; public static final String TEZ_PB_BINARY_CONF_NAME = "tez-conf.pb"; public static final String TEZ_PB_PLAN_BINARY_NAME = "tez-dag.pb"; public static final String TEZ_PB_PLAN_TEXT_NAME = "tez-dag.pb.txt"; diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java b/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java index 3711b5dec4..d6481a83c5 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java @@ -22,7 +22,7 @@ import javax.annotation.Nullable; -import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import com.google.common.annotations.VisibleForTesting; @@ -31,12 +31,17 @@ * Wrapper class to hold user payloads * Provides a version to help in evolving the payloads */ -@Public +@InterfaceAudience.Public public final class UserPayload { - private final ByteBuffer payload; - private final int version; + private ByteBuffer payload; + private int version; private static final ByteBuffer EMPTY_BYTE = ByteBuffer.wrap(new byte[0]); + /** + * Public constructor to allow this descriptor to be instantiated by Jackson. + */ + public UserPayload() {} + private UserPayload(@Nullable ByteBuffer payload) { this(payload, 0); } diff --git a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncherDescriptor.java b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncherDescriptor.java index ff3c90e9ec..39a99cb9b0 100644 --- a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncherDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncherDescriptor.java @@ -22,6 +22,12 @@ @InterfaceStability.Unstable public class ContainerLauncherDescriptor extends NamedEntityDescriptor { + /** + * Public constructor to allow this descriptor to be instantiated by Jackson. + */ + @InterfaceAudience.Private + public ContainerLauncherDescriptor() {} + private ContainerLauncherDescriptor(String containerLauncherName, String containerLauncherClassname) { super(containerLauncherName, containerLauncherClassname); } diff --git a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginsDescriptor.java b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginsDescriptor.java index 89083a0add..fd199fe76a 100644 --- a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginsDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginsDescriptor.java @@ -29,13 +29,16 @@ @InterfaceStability.Unstable public class ServicePluginsDescriptor { - private final boolean enableContainers; - private final boolean enableUber; + private boolean enableContainers; + private boolean enableUber; private TaskSchedulerDescriptor[] taskSchedulerDescriptors; private ContainerLauncherDescriptor[] containerLauncherDescriptors; private TaskCommunicatorDescriptor[] taskCommunicatorDescriptors; + @InterfaceAudience.Private + public ServicePluginsDescriptor() {} + private ServicePluginsDescriptor(boolean enableContainers, boolean enableUber, TaskSchedulerDescriptor[] taskSchedulerDescriptors, ContainerLauncherDescriptor[] containerLauncherDescriptors, diff --git a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicatorDescriptor.java b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicatorDescriptor.java index 57ac385fca..717887dad2 100644 --- a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicatorDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicatorDescriptor.java @@ -22,6 +22,11 @@ @InterfaceStability.Unstable public class TaskCommunicatorDescriptor extends NamedEntityDescriptor { + /** + * Public constructor to allow this descriptor to be instantiated by Jackson. + */ + @InterfaceAudience.Private + public TaskCommunicatorDescriptor() {} private TaskCommunicatorDescriptor(String taskCommName, String taskCommClassname) { super(taskCommName, taskCommClassname); diff --git a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskSchedulerDescriptor.java b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskSchedulerDescriptor.java index 12e091930e..7c7cc4fae0 100644 --- a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskSchedulerDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskSchedulerDescriptor.java @@ -22,6 +22,12 @@ @InterfaceStability.Unstable public class TaskSchedulerDescriptor extends NamedEntityDescriptor { + /** + * Public constructor to allow this descriptor to be instantiated by Jackson. + */ + @InterfaceAudience.Private + public TaskSchedulerDescriptor() { } + private TaskSchedulerDescriptor(String taskSchedulerName, String schedulerClassname) { super(taskSchedulerName, schedulerClassname); } diff --git a/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java b/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java index d7cfcfb55a..661d77320e 100644 --- a/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java +++ b/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java @@ -21,6 +21,7 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.lang.management.ManagementFactory; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -74,14 +75,21 @@ public final class TezUtilsInternal { private TezUtilsInternal() {} - public static ConfigurationProto readUserSpecifiedTezConfiguration(String baseDir) throws - IOException { + public static ConfigurationProto readUserSpecifiedTezConfiguration(String baseDir) throws IOException { File confPBFile = new File(baseDir, TezConstants.TEZ_PB_BINARY_CONF_NAME); try (FileInputStream fis = new FileInputStream(confPBFile)) { return ConfigurationProto.parseFrom(fis); } } + public static Configuration readTezConfigurationXml(InputStream is) { + Configuration configuration = new Configuration(); + if (is != null) { + configuration.addResource(is); + } + return configuration; + } + public static void addUserSpecifiedTezConfiguration(Configuration conf, List kvPairList) { if (kvPairList != null && !kvPairList.isEmpty()) { diff --git a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java index 0d862efdde..146a2dbcf2 100644 --- a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java +++ b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java @@ -17,18 +17,28 @@ package org.apache.tez.common; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.io.InputStream; import java.util.BitSet; import java.util.HashMap; import java.util.Map; import java.util.Random; import org.apache.hadoop.conf.Configuration; +import org.apache.tez.client.TezClientUtils; +import org.apache.tez.dag.api.TezConfiguration; +import org.apache.tez.dag.api.TezConstants; import org.apache.tez.dag.api.UserPayload; import org.apache.tez.dag.api.records.DAGProtos; +import org.apache.tez.serviceplugins.api.ContainerLauncherDescriptor; +import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor; +import org.apache.tez.serviceplugins.api.TaskCommunicatorDescriptor; +import org.apache.tez.serviceplugins.api.TaskSchedulerDescriptor; import com.google.protobuf.ByteString; @@ -291,4 +301,30 @@ public void testPopulateConfProtoFromEntries() { assertEquals(confBuilder.getConfKeyValuesList().size(), 1); } + @Test(timeout = 5000) + public void testReadTezConfigurationXmlFromClasspath() throws IOException { + InputStream is = ClassLoader.getSystemResourceAsStream(TezConfiguration.TEZ_SITE_XML); + Configuration conf = TezUtilsInternal.readTezConfigurationXml(is); + assertEquals("tez.tar.gz", conf.get("tez.lib.uris")); + } + + @Test(timeout = 5000) + public void testPluginsDescriptorFromJSON() throws IOException { + InputStream is = ClassLoader.getSystemResourceAsStream(TezConstants.SERVICE_PLUGINS_DESCRIPTOR_JSON); + ServicePluginsDescriptor spd = TezClientUtils.createPluginsDescriptorFromJSON(is); + TaskSchedulerDescriptor tsd = spd.getTaskSchedulerDescriptors()[0]; + ContainerLauncherDescriptor cld = spd.getContainerLauncherDescriptors()[0]; + TaskCommunicatorDescriptor tcd = spd.getTaskCommunicatorDescriptors()[0]; + + assertFalse(spd.areContainersEnabled()); + assertTrue(spd.isUberEnabled()); + assertEquals("testScheduler0_class", tsd.getClassName()); + assertEquals("testScheduler0", tsd.getEntityName()); + assertEquals("testLauncher0_class", cld.getClassName()); + assertEquals("testLauncher0", cld.getEntityName()); + assertEquals("testComm0_class", tcd.getClassName()); + assertEquals("testComm0", tcd.getEntityName()); + assertEquals(1, tcd.getUserPayload().getVersion()); + assertArrayEquals(new byte[] {0, 0, 0, 1}, tcd.getUserPayload().deepCopyAsArray()); + } } diff --git a/tez-common/src/test/resources/service_plugins_descriptor.json b/tez-common/src/test/resources/service_plugins_descriptor.json new file mode 100644 index 0000000000..31eb1274cb --- /dev/null +++ b/tez-common/src/test/resources/service_plugins_descriptor.json @@ -0,0 +1,26 @@ +{ + "taskSchedulerDescriptors": [ + { + "className": "testScheduler0_class", + "entityName": "testScheduler0" + } + ], + "containerLauncherDescriptors": [ + { + "className": "testLauncher0_class", + "entityName": "testLauncher0" + } + ], + "taskCommunicatorDescriptors": [ + { + "userPayload": { + "payload": "AAAAAQ==", + "version": 1 + }, + "className": "testComm0_class", + "entityName": "testComm0" + } + ], + "enableContainers": false, + "enableUber": true +} diff --git a/tez-common/src/test/resources/tez-site.xml b/tez-common/src/test/resources/tez-site.xml new file mode 100644 index 0000000000..db2b2ab3bb --- /dev/null +++ b/tez-common/src/test/resources/tez-site.xml @@ -0,0 +1,22 @@ + + + + + + + tez.lib.uris + tez.tar.gz + + \ No newline at end of file