[AMBARI-22875] Blueprint cluster creation using manually installed mpacks

ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java

@@ -45,7 +45,6 @@
 import org.apache.ambari.server.controller.internal.ExportBlueprintRequest;
 import org.apache.ambari.server.controller.internal.RequestImpl;
 import org.apache.ambari.server.controller.internal.ResourceImpl;
-import org.apache.ambari.server.controller.internal.Stack;
 import org.apache.ambari.server.controller.spi.ClusterController;
 import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
 import org.apache.ambari.server.controller.spi.NoSuchResourceException;
@@ -56,6 +55,7 @@
 import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
 import org.apache.ambari.server.controller.utilities.PredicateBuilder;
 import org.apache.ambari.server.state.SecurityType;
+import org.apache.ambari.server.state.StackId;
 import org.apache.ambari.server.topology.AmbariContext;
 import org.apache.ambari.server.topology.ClusterTopology;
 import org.apache.ambari.server.topology.ClusterTopologyImpl;
@@ -68,6 +68,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.collect.Iterables;
+
 /**
  * Renderer which renders a cluster resource as a blueprint.
  */
@@ -195,9 +197,12 @@ private Resource createBlueprintResource(TreeNode<Resource> clusterNode) {
     BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology);
     configProcessor.doUpdateForBlueprintExport();
 
-    Stack stack = topology.getBlueprint().getStack();
-    blueprintResource.setProperty("Blueprints/stack_name", stack.getName());
-    blueprintResource.setProperty("Blueprints/stack_version", stack.getVersion());
+    Set<StackId> stackIds = topology.getBlueprint().getStackIds();
+    if (stackIds.size() == 1) {
+      StackId stackId = Iterables.getOnlyElement(stackIds);
+      blueprintResource.setProperty("Blueprints/stack_name", stackId.getStackName());
+      blueprintResource.setProperty("Blueprints/stack_version", stackId.getStackVersion());
+    }
 
     if (topology.isClusterKerberosEnabled()) {
       Map<String, Object> securityConfigMap = new LinkedHashMap<>();

ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java

@@ -18,7 +18,6 @@
 
 package org.apache.ambari.server.api.services.stackadvisor;
 
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -29,7 +28,7 @@
 import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
 import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse.BlueprintConfigurations;
 import org.apache.ambari.server.controller.internal.ConfigurationTopologyException;
-import org.apache.ambari.server.controller.internal.Stack;
+import org.apache.ambari.server.state.StackId;
 import org.apache.ambari.server.state.ValueAttributesInfo;
 import org.apache.ambari.server.topology.AdvisedConfiguration;
 import org.apache.ambari.server.topology.Blueprint;
@@ -77,26 +76,27 @@ public static void init(StackAdvisorHelper instance) {
    * @param userProvidedConfigurations User configurations of cluster provided in Blueprint + Cluster template
    */
   public void adviseConfiguration(ClusterTopology clusterTopology, Map<String, Map<String, String>> userProvidedConfigurations) throws ConfigurationTopologyException {
-    StackAdvisorRequest request = createStackAdvisorRequest(clusterTopology, StackAdvisorRequestType.CONFIGURATIONS);
-    try {
-      RecommendationResponse response = stackAdvisorHelper.recommend(request);
-      addAdvisedConfigurationsToTopology(response, clusterTopology, userProvidedConfigurations);
-    } catch (StackAdvisorException e) {
-      throw new ConfigurationTopologyException(RECOMMENDATION_FAILED, e);
-    } catch (IllegalArgumentException e) {
-      throw new ConfigurationTopologyException(INVALID_RESPONSE, e);
+    for (StackId stackId : clusterTopology.getBlueprint().getStackIds()) {
+      StackAdvisorRequest request = createStackAdvisorRequest(clusterTopology, stackId, StackAdvisorRequestType.CONFIGURATIONS);
+      try {
+        RecommendationResponse response = stackAdvisorHelper.recommend(request);
+        addAdvisedConfigurationsToTopology(response, clusterTopology, userProvidedConfigurations);
+      } catch (StackAdvisorException e) {
+        throw new ConfigurationTopologyException(RECOMMENDATION_FAILED, e);
+      } catch (IllegalArgumentException e) {
+        throw new ConfigurationTopologyException(INVALID_RESPONSE, e);
+      }
     }
   }
 
-  private StackAdvisorRequest createStackAdvisorRequest(ClusterTopology clusterTopology, StackAdvisorRequestType requestType) {
-    Stack stack = clusterTopology.getBlueprint().getStack(); // TODO: implement multi-stack
+  private StackAdvisorRequest createStackAdvisorRequest(ClusterTopology clusterTopology, StackId stackId, StackAdvisorRequestType requestType) {
     Map<String, Set<String>> hgComponentsMap = gatherHostGroupComponents(clusterTopology);
     Map<String, Set<String>> hgHostsMap = gatherHostGroupBindings(clusterTopology);
     Map<String, Set<String>> componentHostsMap = gatherComponentsHostsMap(hgComponentsMap,
             hgHostsMap);
     return StackAdvisorRequest.StackAdvisorRequestBuilder
-      .forStack(stack.getName(), stack.getVersion())
-      .forServices(new ArrayList<>(clusterTopology.getBlueprint().getServices()))
+      .forStack(stackId)
+      .forServices(clusterTopology.getBlueprint().getStack().getServices(stackId))
       .forHosts(gatherHosts(clusterTopology))
       .forHostsGroupBindings(gatherHostGroupBindings(clusterTopology))
       .forHostComponents(gatherHostGroupComponents(clusterTopology))

ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java

@@ -33,7 +33,6 @@
 import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
 import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
 import org.apache.ambari.server.configuration.Configuration;
-
 import org.apache.ambari.server.state.ServiceInfo;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -122,9 +121,7 @@ public synchronized RecommendationResponse recommend(StackAdvisorRequest request
       throws StackAdvisorException {
       requestId = generateRequestId();
 
-    // TODO, need to pass the service Name that was modified.
-    // For now, hardcode
-    String serviceName = "ZOOKEEPER";
+    String serviceName = request.getServices().stream().findAny().orElse(null);
 
     ServiceInfo.ServiceAdvisorType serviceAdvisorType = getServiceAdvisorType(request.getStackName(), request.getStackVersion(), serviceName);
     StackAdvisorCommand<RecommendationResponse> command = createRecommendationCommand(serviceName, request);

ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java

@@ -29,6 +29,7 @@
 
 import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
 import org.apache.ambari.server.state.ChangedConfigInfo;
+import org.apache.ambari.server.state.StackId;
 import org.apache.commons.lang.StringUtils;
 
 import com.google.common.base.Preconditions;
@@ -146,6 +147,10 @@ public static StackAdvisorRequestBuilder forStack(String stackName, String stack
       return new StackAdvisorRequestBuilder(stackName, stackVersion);
     }
 
+    public static StackAdvisorRequestBuilder forStack(StackId stackId) {
+      return new StackAdvisorRequestBuilder(stackId.getStackName(), stackId.getStackVersion());
+    }
+
     public StackAdvisorRequestBuilder ofType(StackAdvisorRequestType requestType) {
       this.instance.requestType = requestType;
       return this;

ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRunner.java

@@ -52,7 +52,7 @@ public class StackAdvisorRunner {
    */
   public void runScript(ServiceInfo.ServiceAdvisorType serviceAdvisorType, StackAdvisorCommandType saCommandType, File actionDirectory)
       throws StackAdvisorException {
-    LOG.info(String.format("StackAdvisorRunner. serviceAdvisorType=%s, actionDirectory=%s, command=%s", serviceAdvisorType.toString(), actionDirectory,
+    LOG.info(String.format("StackAdvisorRunner. serviceAdvisorType=%s, actionDirectory=%s, command=%s", serviceAdvisorType, actionDirectory,
         saCommandType));
 
     String outputFile = actionDirectory + File.separator + "stackadvisor.out";

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java

@@ -63,7 +63,6 @@
 import org.apache.ambari.server.controller.internal.AbstractControllerResourceProvider;
 import org.apache.ambari.server.controller.internal.AmbariPrivilegeResourceProvider;
 import org.apache.ambari.server.controller.internal.BaseClusterRequest;
-import org.apache.ambari.server.controller.internal.BlueprintResourceProvider;
 import org.apache.ambari.server.controller.internal.ClusterPrivilegeResourceProvider;
 import org.apache.ambari.server.controller.internal.ClusterResourceProvider;
 import org.apache.ambari.server.controller.internal.HostResourceProvider;
@@ -940,8 +939,6 @@ public void performStaticInjection() {
     SecurityFilter.init(injector.getInstance(Configuration.class));
     StackDefinedPropertyProvider.init(injector);
     AbstractControllerResourceProvider.init(injector.getInstance(ResourceProviderFactory.class));
-    BlueprintResourceProvider.init(injector.getInstance(BlueprintFactory.class),
-        injector.getInstance(BlueprintDAO.class), injector.getInstance(SecurityConfigurationFactory.class), ambariMetaInfo);
     StackDependencyResourceProvider.init(ambariMetaInfo);
     ClusterResourceProvider.init(injector.getInstance(TopologyManager.class),
         injector.getInstance(TopologyRequestFactoryImpl.class), injector.getInstance(SecurityConfigurationFactory

ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java

@@ -63,6 +63,7 @@
 import org.apache.ambari.server.configuration.Configuration.ConnectionPoolType;
 import org.apache.ambari.server.configuration.Configuration.DatabaseType;
 import org.apache.ambari.server.controller.internal.AlertTargetResourceProvider;
+import org.apache.ambari.server.controller.internal.BlueprintResourceProvider;
 import org.apache.ambari.server.controller.internal.ClusterSettingResourceProvider;
 import org.apache.ambari.server.controller.internal.ClusterStackVersionResourceProvider;
 import org.apache.ambari.server.controller.internal.ComponentResourceProvider;
@@ -168,6 +169,8 @@
 import org.apache.ambari.server.state.stack.OsFamily;
 import org.apache.ambari.server.state.svccomphost.ServiceComponentHostImpl;
 import org.apache.ambari.server.topology.BlueprintFactory;
+import org.apache.ambari.server.topology.BlueprintValidator;
+import org.apache.ambari.server.topology.BlueprintValidatorImpl;
 import org.apache.ambari.server.topology.PersistedState;
 import org.apache.ambari.server.topology.PersistedStateImpl;
 import org.apache.ambari.server.topology.SecurityConfigurationFactory;
@@ -506,6 +509,7 @@ private void installFactories() {
         .implement(ResourceProvider.class, Names.named("alertTarget"), AlertTargetResourceProvider.class)
         .implement(ResourceProvider.class, Names.named("viewInstance"), ViewInstanceResourceProvider.class)
         .implement(ResourceProvider.class, Names.named("rootServiceHostComponentConfiguration"), RootServiceComponentConfigurationResourceProvider.class)
+        .implement(ResourceProvider.class, Names.named(BlueprintResourceProvider.NAME), BlueprintResourceProvider.class)
         .build(ResourceProviderFactory.class));
 
     install(new FactoryModuleBuilder().implement(
@@ -537,6 +541,7 @@ private void installFactories() {
     bind(RegistryFactory.class).to(RegistryFactoryImpl.class);
     bind(HostRoleCommandFactory.class).to(HostRoleCommandFactoryImpl.class);
     bind(SecurityHelper.class).toInstance(SecurityHelperImpl.getInstance());
+    bind(BlueprintValidator.class).to(BlueprintValidatorImpl.class);
     bind(BlueprintFactory.class);
 
     install(new FactoryModuleBuilder().implement(AmbariEvent.class, Names.named("userCreated"), UserCreatedEvent.class).build(AmbariEventFactory.class));

ambari-server/src/main/java/org/apache/ambari/server/controller/ResourceProviderFactory.java

@@ -22,6 +22,7 @@
 import javax.inject.Named;
 
 import org.apache.ambari.server.controller.internal.AlertTargetResourceProvider;
+import org.apache.ambari.server.controller.internal.BlueprintResourceProvider;
 import org.apache.ambari.server.controller.internal.ClusterStackVersionResourceProvider;
 import org.apache.ambari.server.controller.internal.UpgradeResourceProvider;
 import org.apache.ambari.server.controller.internal.ViewInstanceResourceProvider;
@@ -89,4 +90,9 @@ public interface ResourceProviderFactory {
   @Named("viewInstance")
   ViewInstanceResourceProvider getViewInstanceResourceProvider();
 
+  @Named(BlueprintResourceProvider.NAME)
+  BlueprintResourceProvider getBlueprintResourceProvider(
+    AmbariManagementController managementController
+  );
+
 }

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java

@@ -226,7 +226,7 @@ public static ResourceProvider getResourceProvider(Resource.Type type,
       case HostComponentProcess:
         return new HostComponentProcessResourceProvider(managementController);
       case Blueprint:
-        return new BlueprintResourceProvider(managementController);
+        return resourceProviderFactory.getBlueprintResourceProvider(managementController);
       case KerberosDescriptor:
         return resourceProviderFactory.getKerberosDescriptorResourceProvider(managementController);
       case Recommendation:

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java

@@ -58,13 +58,15 @@
 
 import com.google.common.base.Predicates;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 
 /**
  * Updates configuration properties based on cluster topology.  This is done when exporting
  * a blueprint and when a cluster is provisioned via a blueprint.
  */
+// TODO move to topology package
 public class BlueprintConfigurationProcessor {
 
   private static final Logger LOG = LoggerFactory.getLogger(BlueprintConfigurationProcessor.class);
@@ -437,7 +439,7 @@ public Set<String> doUpdateForClusterCreate() throws ConfigurationTopologyExcept
 
   private void trimProperties(Configuration clusterConfig, ClusterTopology clusterTopology) {
     Blueprint blueprint = clusterTopology.getBlueprint();
-    Stack stack = blueprint.getStack();
+    StackDefinition stack = blueprint.getStack();
 
     Map<String, Map<String, String>> configTypes = clusterConfig.getFullProperties();
     for (String configType : configTypes.keySet()) {
@@ -448,7 +450,7 @@ private void trimProperties(Configuration clusterConfig, ClusterTopology cluster
     }
   }
 
-  private void trimPropertyValue(Configuration clusterConfig, Stack stack, String configType, Map<String, String> properties, String propertyName) {
+  private void trimPropertyValue(Configuration clusterConfig, StackDefinition stack, String configType, Map<String, String> properties, String propertyName) {
     if (propertyName != null && properties.get(propertyName) != null) {
 
       TrimmingStrategy trimmingStrategy =
@@ -2880,7 +2882,7 @@ private Collection<String> setupHDFSProxyUsers(Configuration configuration, Set<
    * @param configTypesUpdated
    * @param stack
    */
-  private void addExcludedConfigProperties(Configuration configuration, Set<String> configTypesUpdated, Stack stack) {
+  private void addExcludedConfigProperties(Configuration configuration, Set<String> configTypesUpdated, StackDefinition stack) {
     Collection<String> blueprintServices = clusterTopology.getBlueprint().getServices();
 
     LOG.debug("Handling excluded properties for blueprint services: {}", blueprintServices);
@@ -2971,38 +2973,41 @@ private static void setRetryConfiguration(Configuration configuration, Set<Strin
    * @param configTypesUpdated
    *          the list of configuration types updated (cluster-env will be added
    *          to this).
-   * @throws ConfigurationTopologyException
    */
   private void setStackToolsAndFeatures(Configuration configuration, Set<String> configTypesUpdated)
       throws ConfigurationTopologyException {
     ConfigHelper configHelper = clusterTopology.getAmbariContext().getConfigHelper();
-    Stack stack = clusterTopology.getBlueprint().getStack();
-    String stackName = stack.getName();
-    String stackVersion = stack.getVersion();
 
-    StackId stackId = new StackId(stackName, stackVersion);
-
-    Set<String> properties = Sets.newHashSet(ConfigHelper.CLUSTER_ENV_STACK_NAME_PROPERTY,
-        ConfigHelper.CLUSTER_ENV_STACK_ROOT_PROPERTY, ConfigHelper.CLUSTER_ENV_STACK_TOOLS_PROPERTY,
-        ConfigHelper.CLUSTER_ENV_STACK_FEATURES_PROPERTY,
-        ConfigHelper.CLUSTER_ENV_STACK_PACKAGES_PROPERTY);
+    Set<String> properties = ImmutableSet.of(
+      ConfigHelper.CLUSTER_ENV_STACK_NAME_PROPERTY,
+      ConfigHelper.CLUSTER_ENV_STACK_ROOT_PROPERTY,
+      ConfigHelper.CLUSTER_ENV_STACK_TOOLS_PROPERTY,
+      ConfigHelper.CLUSTER_ENV_STACK_FEATURES_PROPERTY,
+      ConfigHelper.CLUSTER_ENV_STACK_PACKAGES_PROPERTY
+    );
 
     try {
-      Map<String, Map<String, String>> defaultStackProperties = configHelper.getDefaultStackProperties(stackId);
-      Map<String,String> clusterEnvDefaultProperties = defaultStackProperties.get(CLUSTER_ENV_CONFIG_TYPE_NAME);
-
-      for( String property : properties ){
-        if (clusterEnvDefaultProperties.containsKey(property)) {
-          configuration.setProperty(CLUSTER_ENV_CONFIG_TYPE_NAME, property,
-              clusterEnvDefaultProperties.get(property));
+      for (StackId stackId : clusterTopology.getBlueprint().getStackIds()) {
+        Map<String, Map<String, String>> defaultStackProperties = configHelper.getDefaultStackProperties(stackId);
+        if (defaultStackProperties.containsKey(CLUSTER_ENV_CONFIG_TYPE_NAME)) {
+          Map<String, String> clusterEnvDefaultProperties = defaultStackProperties.get(CLUSTER_ENV_CONFIG_TYPE_NAME);
+
+          for (String property : properties) {
+            if (clusterEnvDefaultProperties.containsKey(property)) {
+              configuration.setProperty(CLUSTER_ENV_CONFIG_TYPE_NAME, property,
+                clusterEnvDefaultProperties.get(property)
+              );
+
+              // make sure to include the configuration type as being updated
+              configTypesUpdated.add(CLUSTER_ENV_CONFIG_TYPE_NAME);
+            }
+          }
 
-          // make sure to include the configuration type as being updated
-          configTypesUpdated.add(CLUSTER_ENV_CONFIG_TYPE_NAME);
+          break;
         }
       }
-    } catch( AmbariException ambariException ){
-      throw new ConfigurationTopologyException("Unable to retrieve the stack tools and features",
-          ambariException);
+    } catch (AmbariException e) {
+      throw new ConfigurationTopologyException("Unable to retrieve the stack tools and features", e);
     }
   }
 
@@ -3101,7 +3106,7 @@ private static class StackPropertyTypeFilter implements PropertyFilter {
      */
     @Override
     public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType, ClusterTopology topology) {
-        Stack stack = topology.getBlueprint().getStack();
+        StackDefinition stack = topology.getBlueprint().getStack();
         final String serviceName = stack.getServiceForConfigType(configType);
         return !(stack.isPasswordProperty(serviceName, configType, propertyName) ||
                 stack.isKerberosPrincipalNameProperty(serviceName, configType, propertyName));
@@ -3198,7 +3203,7 @@ private static abstract class DependencyFilter implements PropertyFilter {
      */
     @Override
     public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType, ClusterTopology topology) {
-      Stack stack = topology.getBlueprint().getStack();
+      StackDefinition stack = topology.getBlueprint().getStack();
       Configuration configuration = topology.getConfiguration();
 
       final String serviceName = stack.getServiceForConfigType(configType);