AMBARI-22791 support for parsing MulitE blueprints and cluster templates

ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java

@@ -59,7 +59,6 @@
 import org.apache.ambari.server.topology.AmbariContext;
 import org.apache.ambari.server.topology.ClusterTopology;
 import org.apache.ambari.server.topology.ClusterTopologyImpl;
-import org.apache.ambari.server.topology.Component;
 import org.apache.ambari.server.topology.Configuration;
 import org.apache.ambari.server.topology.HostGroup;
 import org.apache.ambari.server.topology.HostGroupInfo;
@@ -445,10 +444,10 @@ private List<Map<String, Object>> formatGroupsAsList(ClusterTopology topology) {
    */
   private List<Map<String, String>> processHostGroupComponents(HostGroup group) {
     List<Map<String, String>> listHostGroupComponents = new ArrayList<>();
-    for (Component component : group.getComponents()) {
+    for (String component : group.getComponentNames()) {
       Map<String, String> mapComponentProperties = new HashMap<>();
       listHostGroupComponents.add(mapComponentProperties);
-      mapComponentProperties.put("name", component.getName());
+      mapComponentProperties.put("name", component);
     }
     return listHostGroupComponents;
   }

ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java

@@ -89,7 +89,7 @@ public void adviseConfiguration(ClusterTopology clusterTopology, Map<String, Map
   }
 
   private StackAdvisorRequest createStackAdvisorRequest(ClusterTopology clusterTopology, StackAdvisorRequestType requestType) {
-    Stack stack = clusterTopology.getBlueprint().getStack();
+    Stack stack = clusterTopology.getBlueprint().getStack(); // TODO: implement multi-stack
     Map<String, Set<String>> hgComponentsMap = gatherHostGroupComponents(clusterTopology);
     Map<String, Set<String>> hgHostsMap = gatherHostGroupBindings(clusterTopology);
     Map<String, Set<String>> componentHostsMap = gatherComponentsHostsMap(hgComponentsMap,

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java

@@ -941,8 +941,7 @@ public void performStaticInjection() {
     StackDefinedPropertyProvider.init(injector);
     AbstractControllerResourceProvider.init(injector.getInstance(ResourceProviderFactory.class));
     BlueprintResourceProvider.init(injector.getInstance(BlueprintFactory.class),
-        injector.getInstance(BlueprintDAO.class), injector.getInstance(SecurityConfigurationFactory.class),
-        injector.getInstance(Gson.class), ambariMetaInfo);
+        injector.getInstance(BlueprintDAO.class), injector.getInstance(SecurityConfigurationFactory.class), ambariMetaInfo);
     StackDependencyResourceProvider.init(ambariMetaInfo);
     ClusterResourceProvider.init(injector.getInstance(TopologyManager.class),
         injector.getInstance(TopologyRequestFactoryImpl.class), injector.getInstance(SecurityConfigurationFactory

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java

@@ -18,7 +18,12 @@
 
 package org.apache.ambari.server.controller.internal;
 
+import static java.util.stream.Collectors.toList;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -57,17 +62,18 @@
 import org.apache.ambari.server.topology.BlueprintFactory;
 import org.apache.ambari.server.topology.GPLLicenseNotAcceptedException;
 import org.apache.ambari.server.topology.InvalidTopologyException;
+import org.apache.ambari.server.topology.MpackInstance;
 import org.apache.ambari.server.topology.SecurityConfiguration;
 import org.apache.ambari.server.topology.SecurityConfigurationFactory;
 import org.apache.ambari.server.utils.SecretReference;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Sets;
-import com.google.gson.Gson;
 
 
 /**
@@ -101,6 +107,8 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
   public static final String COMPONENT_PROPERTY_ID ="components";
   public static final String COMPONENT_NAME_PROPERTY_ID ="name";
   public static final String COMPONENT_PROVISION_ACTION_PROPERTY_ID = "provision_action";
+  protected static final String COMPONENT_MPACK_INSTANCE_PROPERTY = "mpack_instance";
+  protected static final String COMPONENT_SERVICE_INSTANCE_PROPERTY = "service_instance";
 
   // Configurations
   public static final String CONFIGURATION_PROPERTY_ID = "configurations";
@@ -120,6 +128,12 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
     "Configuration elements must be Maps";
   public static final String CONFIGURATION_MAP_SIZE_CHECK_ERROR_MESSAGE =
     "Configuration Maps must hold a single configuration type each";
+  public static final String MPACK_INSTANCES_PROPERTY_ID = "mpack_instances";
+
+  // Primary Key Fields
+  private static Set<String> pkPropertyIds =
+    new HashSet<>(Arrays.asList(new String[]{
+      BLUEPRINT_NAME_PROPERTY_ID}));
 
   /**
    * The key property ids for a Blueprint resource.
@@ -138,7 +152,8 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
       BLUEPRINT_SECURITY_PROPERTY_ID,
       HOST_GROUP_PROPERTY_ID,
       CONFIGURATION_PROPERTY_ID,
-      SETTING_PROPERTY_ID);
+      SETTING_PROPERTY_ID,
+      MPACK_INSTANCES_PROPERTY_ID);
 
   /**
    * Used to create Blueprint instances
@@ -158,7 +173,7 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
   /**
    * Used to serialize to/from json.
    */
-  private static Gson jsonSerializer;
+  private static ObjectMapper jsonSerializer = new ObjectMapper();
 
   // ----- Constructors ----------------------------------------------------
 
@@ -176,14 +191,12 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
    *
    * @param factory   blueprint factory
    * @param dao       blueprint data access object
-   * @param gson      json serializer
    */
   public static void init(BlueprintFactory factory, BlueprintDAO dao, SecurityConfigurationFactory
-    securityFactory, Gson gson, AmbariMetaInfo metaInfo) {
+    securityFactory, AmbariMetaInfo metaInfo) {
     blueprintFactory = factory;
     blueprintDAO = dao;
     securityConfigurationFactory = securityFactory;
-    jsonSerializer = gson;
     ambariMetaInfo = metaInfo;
   }
 
@@ -325,10 +338,16 @@ protected Resource toResource(BlueprintEntity entity, Set<String> requestedIds)
       for (HostGroupComponentEntity component : components) {
         Map<String, String> mapComponentProps = new HashMap<>();
         mapComponentProps.put(COMPONENT_NAME_PROPERTY_ID, component.getName());
-
         if (component.getProvisionAction() != null) {
           mapComponentProps.put(COMPONENT_PROVISION_ACTION_PROPERTY_ID, component.getProvisionAction().toString());
         }
+        if (component.getMpackName() != null) {
+          mapComponentProps.put(COMPONENT_MPACK_INSTANCE_PROPERTY,
+            component.getMpackName() + "-" + component.getMpackVersion());
+        }
+        if (component.getServiceName() != null) {
+          mapComponentProps.put(COMPONENT_SERVICE_INSTANCE_PROPERTY, component.getServiceName());
+        }
 
         listComponentProps.add(mapComponentProps);
       }
@@ -351,9 +370,37 @@ protected Resource toResource(BlueprintEntity entity, Set<String> requestedIds)
       setResourceProperty(resource, BLUEPRINT_SECURITY_PROPERTY_ID, securityConfigMap, requestedIds);
     }
 
+    Collection<Map<String, Object>> mpacks = entity.getMpackInstances().stream().map(mpackEntity -> {
+      MpackInstance mpack = MpackInstance.fromEntity(mpackEntity);
+      Map<String, Object> mpackAsMap = fromJson(toJson(mpack), Map.class);
+      return mpackAsMap;
+    } ).collect(toList());
+    setResourceProperty(resource, MPACK_INSTANCES_PROPERTY_ID, mpacks, requestedIds);
+
     return resource;
   }
 
+  private static <T> T fromJson(String json, Class<? extends T> valueType) {
+    if (null == json) {
+      return  null;
+    }
+    try {
+      return jsonSerializer.readValue(json, valueType);
+    }
+    catch (IOException ex) {
+      throw new UncheckedIOException(ex);
+    }
+  }
+
+  private static String toJson(Object object) {
+    try {
+      return jsonSerializer.writeValueAsString(object);
+    }
+    catch (IOException ex) {
+      throw new UncheckedIOException(ex);
+    }
+  }
+
   /**
    * Populate a list of configuration property maps from a collection of configuration entities.
    *
@@ -371,9 +418,7 @@ List<Map<String, Map<String, Object>>> populateConfigurationList(
       String type = config.getType();
 
       if(config instanceof BlueprintConfigEntity) {
-        Map<String, String> properties = jsonSerializer.<Map<String, String>>fromJson(
-            config.getConfigData(), Map.class);
-
+        Map<String, String> properties = fromJson(config.getConfigData(), Map.class);
 
         // TODO: use multiple mpacks
         BlueprintMpackInstanceEntity mpack =
@@ -387,19 +432,17 @@ List<Map<String, Map<String, Object>>> populateConfigurationList(
         }
 
         Map<org.apache.ambari.server.state.PropertyInfo.PropertyType, Set<String>> propertiesTypes =
-            metaInfoStack.getConfigPropertiesTypes(type);
+          metaInfoStack.getConfigPropertiesTypes(type);
 
         SecretReference.replacePasswordsWithReferences(propertiesTypes, properties, type, -1l);
 
         configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties);
       } else {
-        Map<String, Object> properties = jsonSerializer.<Map<String, Object>>fromJson(
-            config.getConfigData(), Map.class);
+        Map<String, Object> properties = fromJson(config.getConfigData(), Map.class);
         configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties);
       }
 
-      Map<String, Map<String, String>> attributes = jsonSerializer.<Map<String, Map<String, String>>>fromJson(
-          config.getConfigAttributes(), Map.class);
+      Map<String, Map<String, String>> attributes = fromJson(config.getConfigAttributes(), Map.class);
       if (attributes != null && !attributes.isEmpty()) {
         configTypeDefinition.put(PROPERTIES_ATTRIBUTES_PROPERTY_ID, attributes);
       }
@@ -423,8 +466,7 @@ public static List<Map<String, Object>> populateSettingList(
 
     if (settings != null) {
       for (BlueprintSettingEntity setting : settings) {
-        List<Map<String, String>> propertiesList = jsonSerializer.<List<Map<String, String>>>fromJson(
-                setting.getSettingData(), List.class);
+        List<Map<String, String>> propertiesList = fromJson(setting.getSettingData(), List.class);
         Map<String, Object> settingMap = new HashMap<>();
         settingMap.put(setting.getSettingName(), propertiesList);
         listSettings.add(settingMap);
@@ -503,7 +545,7 @@ public Void invoke() throws AmbariException {
         String rawRequestBody = requestInfoProps.get(Request.REQUEST_INFO_BODY_PROPERTY);
         Preconditions.checkArgument(!Strings.isNullOrEmpty(rawRequestBody), REQUEST_BODY_EMPTY_ERROR_MESSAGE);
 
-        Map<String, Object> rawBodyMap = jsonSerializer.<Map<String, Object>>fromJson(rawRequestBody, Map.class);
+        Map<String, Object> rawBodyMap = fromJson(rawRequestBody, Map.class);
         Object configurationData = rawBodyMap.get(CONFIGURATION_PROPERTY_ID);
 
         if (configurationData != null) {
@@ -545,8 +587,7 @@ public Void invoke() throws AmbariException {
         }
 
         LOG.info("Creating Blueprint, name=" + blueprint.getName());
-        String blueprintSetting = blueprint.getSetting() == null ? "(null)" :
-                jsonSerializer.toJson(blueprint.getSetting().getProperties());
+        String blueprintSetting = blueprint.getSetting() == null ? "(null)" : toJson(blueprint.getSetting().getProperties());
         LOG.info("Blueprint setting=" + blueprintSetting);
 
         try {
@@ -588,8 +629,8 @@ public void applyConfiguration(Map<String, String> configuration, BlueprintConfi
         }
       }
 
-      blueprintConfiguration.setConfigData(jsonSerializer.toJson(configData));
-      blueprintConfiguration.setConfigAttributes(jsonSerializer.toJson(configAttributes));
+      blueprintConfiguration.setConfigData(toJson(configData));
+      blueprintConfiguration.setConfigAttributes(toJson(configAttributes));
     }
 
     protected abstract void addProperty(Map<String, String> configData,

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java

@@ -86,6 +86,7 @@ public class ClusterResourceProvider extends AbstractControllerResourceProvider
   public static final String HEALTH_REPORT = "health_report";
   public static final String CREDENTIAL_STORE_PROPERTIES = "credential_store_properties";
   public static final String REPO_VERSION = "repository_version";
+  public static final String MPACK_INSTANCES = BlueprintResourceProvider.MPACK_INSTANCES_PROPERTY_ID;
   public static final String CLUSTER_ID_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + CLUSTER_ID;
   public static final String CLUSTER_NAME_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + CLUSTER_NAME;
   public static final String CLUSTER_VERSION_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + VERSION;
@@ -163,6 +164,7 @@ public class ClusterResourceProvider extends AbstractControllerResourceProvider
     propertyIds.add(SECURITY);
     propertyIds.add(CREDENTIALS);
     propertyIds.add(QUICKLINKS_PROFILE);
+    propertyIds.add(MPACK_INSTANCES);
     propertyIds.add(CLUSTER_STATE_PROPERTY_ID);
   }
 

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java

@@ -132,10 +132,10 @@ private void createBlueprint(Collection<ExportedHostGroup> exportedHostGroups, S
         componentList.add(new Component(component));
       }
 
-      hostGroups.add(new HostGroupImpl(exportedHostGroup.getName(), bpName, stack, componentList,
+      hostGroups.add(new HostGroupImpl(exportedHostGroup.getName(), bpName, Collections.singleton(stack), componentList,
           exportedHostGroup.getConfiguration(), String.valueOf(exportedHostGroup.getCardinality())));
     }
-    blueprint = new BlueprintImpl(bpName, hostGroups, stack, configuration, null);
+    blueprint = new BlueprintImpl(bpName, hostGroups, stack, configuration, null, null);
   }
 
   private void createHostGroupInfo(Collection<ExportedHostGroup> exportedHostGroups) {

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java

@@ -17,6 +17,9 @@
  */
 package org.apache.ambari.server.controller.internal;
 
+import static java.util.stream.Collectors.toList;
+
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
@@ -34,11 +37,16 @@
 import org.apache.ambari.server.topology.Credential;
 import org.apache.ambari.server.topology.HostGroupInfo;
 import org.apache.ambari.server.topology.InvalidTopologyTemplateException;
+import org.apache.ambari.server.topology.ManagementPackMapping;
+import org.apache.ambari.server.topology.MpackInstance;
 import org.apache.ambari.server.topology.NoSuchBlueprintException;
 import org.apache.ambari.server.topology.SecurityConfiguration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Enums;
 import com.google.common.base.Optional;
 import com.google.common.base.Strings;
@@ -119,6 +127,13 @@ public class ProvisionClusterRequest extends BaseClusterRequest {
    */
   public static final String QUICKLINKS_PROFILE_SERVICES_PROPERTY = "quicklinks_profile/services";
 
+  public static final String MANAGEMENT_PACK_MAPPINGS_PROPERTY = "management_pack_mappings";
+
+  public static final String MPACK_INSTANCES_PROPERTY = BlueprintResourceProvider.MPACK_INSTANCES_PROPERTY_ID;
+
+  public static final String MPACK_INSTANCE_PROPERTY = "mpack_instance";
+
+  public static final String COMPONENT_NAME_PROPERTY = "component_name";
 
   /**
    * configuration factory
@@ -148,6 +163,8 @@ public class ProvisionClusterRequest extends BaseClusterRequest {
 
   private final String quickLinksProfileJson;
 
+  private Collection<MpackInstance> mpackInstances;
+
   private final static Logger LOG = LoggerFactory.getLogger(ProvisionClusterRequest.class);
 
   /**
@@ -196,13 +213,30 @@ public ProvisionClusterRequest(Map<String, Object> properties, SecurityConfigura
 
     setProvisionAction(parseProvisionAction(properties));
 
+    processMpackInstances(properties);
+
     try {
       this.quickLinksProfileJson = processQuickLinksProfile(properties);
     } catch (QuickLinksProfileEvaluationException ex) {
       throw new InvalidTopologyTemplateException("Invalid quick links profile", ex);
     }
   }
 
+  private void processMpackInstances(Map<String, Object> properties) throws InvalidTopologyTemplateException {
+    if (properties.containsKey(MPACK_INSTANCES_PROPERTY)) {
+      ObjectMapper mapper = new ObjectMapper();
+      mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+      try {
+        String mpackInstancesJson = mapper.writeValueAsString(properties.get(MPACK_INSTANCES_PROPERTY));
+        this.mpackInstances = mapper.readValue(mpackInstancesJson,
+          new TypeReference<Collection<MpackInstance>>() {});
+      }
+      catch (IOException ex) {
+        throw new InvalidTopologyTemplateException("Cannot process mpack instances.", ex);
+      }
+    }
+  }
+
   private String processQuickLinksProfile(Map<String, Object> properties) throws QuickLinksProfileEvaluationException {
     Object globalFilters = properties.get(QUICKLINKS_PROFILE_FILTERS_PROPERTY);
     Object serviceFilters = properties.get(QUICKLINKS_PROFILE_SERVICES_PROPERTY);
@@ -339,6 +373,16 @@ private void processHostGroup(Map<String, Object> hostGroupProperties) throws In
     processGroupHosts(name, (Collection<Map<String, String>>)
       hostGroupProperties.get(HOSTGROUP_HOSTS_PROPERTY), hostGroupInfo);
 
+    // process mpack mappings
+    if (hostGroupProperties.containsKey(MANAGEMENT_PACK_MAPPINGS_PROPERTY)) {
+      Set<Map<String, String>> mpackMappingsRaw =
+        (Set<Map<String, String>>)hostGroupProperties.get(MANAGEMENT_PACK_MAPPINGS_PROPERTY);
+      Collection<ManagementPackMapping> mpackMappings = mpackMappingsRaw.stream().
+        map(mapping -> new ManagementPackMapping(mapping.get(COMPONENT_NAME_PROPERTY), mapping.get(MPACK_INSTANCE_PROPERTY))).
+        collect(toList());
+      hostGroupInfo.setManagementPackMappings(mpackMappings);
+    }
+
     // don't set the parent configuration
     hostGroupInfo.setConfiguration(configurationFactory.getConfiguration(
       (Collection<Map<String, String>>) hostGroupProperties.get(CONFIGURATIONS_PROPERTY)));
@@ -490,4 +534,7 @@ public String getDefaultPassword() {
     return defaultPassword;
   }
 
+  public Collection<MpackInstance> getMpackInstances() {
+    return mpackInstances;
+  }
 }