Skip to content

[AMBARI-23032] fix exceptions, export cluster settings and single mpack #431

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
benyoka merged 8 commits into from
Mar 1, 2018
Merged

[AMBARI-23032] fix exceptions, export cluster settings and single mpack #431

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
63083ec
AMBARI-23032 fix execptions, export cluster settings and single mpack…
Feb 21, 2018
4ec0391
AMBARI-23032 fix review comments (benyoka)
Feb 22, 2018
bd2f0b3
AMBARI-23032 better method naming (benyoka)
Feb 22, 2018
33c0f20
AMBARI-23032 fix review comments #2 (benyoka)
Feb 22, 2018
aba404a
AMBARI-23032 fix review comments #3 (benyoka)
Feb 22, 2018
c31829c
Merge remote-tracking branch 'upstream/branch-feature-AMBARI-14714' i…
Feb 26, 2018
91f1560
AMBARI-23032 fix swallowed exceptions during mpack installation
Feb 26, 2018
eb370a6
Merge branch 'branch-feature-AMBARI-14714' into AMBARI-23032-branch-f…
adoroszlai Mar 1, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 75 additions & 69 deletions ...ver/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@

package org.apache.ambari.server.api.query.render;

import static java.util.stream.Collectors.toList;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
Expand All @@ -26,10 +28,12 @@
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;

import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.query.QueryInfo;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.api.services.Request;
import org.apache.ambari.server.api.services.Result;
import org.apache.ambari.server.api.services.ResultImpl;
Expand All @@ -42,6 +46,7 @@
import org.apache.ambari.server.controller.internal.ArtifactResourceProvider;
import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor;
import org.apache.ambari.server.controller.internal.BlueprintResourceProvider;
import org.apache.ambari.server.controller.internal.ClusterSettingResourceProvider;
import org.apache.ambari.server.controller.internal.ExportBlueprintRequest;
import org.apache.ambari.server.controller.internal.RequestImpl;
import org.apache.ambari.server.controller.internal.ResourceImpl;
Expand All @@ -68,7 +73,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.collect.Iterables;
import com.google.common.collect.ImmutableMap;

/**
* Renderer which renders a cluster resource as a blueprint.
Expand All @@ -80,6 +85,12 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer {
*/
private AmbariManagementController controller = AmbariServer.getController();


/**
* MetaInfo used to get stack and mpack information.
*/
private AmbariMetaInfo metaInfo = controller.getAmbariMetaInfo();

// /**
// * Map of configuration type to configuration properties which are required that a user
// * input. These properties will be stripped from the exported blueprint.
Expand All @@ -101,48 +112,38 @@ public TreeNode<Set<String>> finalizeProperties(

copyPropertiesToResult(queryProperties, resultTree);

String configType = Resource.Type.Configuration.name();
if (resultTree.getChild(configType) == null) {
resultTree.addChild(new HashSet<>(), configType);
}
ensureChild(resultTree, Resource.Type.Configuration.name(), "properties");

String serviceType = Resource.Type.Service.name();
if (resultTree.getChild(serviceType) == null) {
resultTree.addChild(new HashSet<>(), serviceType);
}
TreeNode<Set<String>> serviceNode = resultTree.getChild(serviceType);
if (serviceNode == null) {
serviceNode = resultTree.addChild(new HashSet<>(), serviceType);
}
String serviceComponentType = Resource.Type.Component.name();
TreeNode<Set<String>> serviceComponentNode = resultTree.getChild(
serviceType + "/" + serviceComponentType);
if (serviceComponentNode == null) {
serviceComponentNode = serviceNode.addChild(new HashSet<>(), serviceComponentType);
}
serviceComponentNode.getObject().add("ServiceComponentInfo/cluster_name");
serviceComponentNode.getObject().add("ServiceComponentInfo/service_name");
serviceComponentNode.getObject().add("ServiceComponentInfo/component_name");
serviceComponentNode.getObject().add("ServiceComponentInfo/recovery_enabled");

String hostType = Resource.Type.Host.name();
String hostComponentType = Resource.Type.HostComponent.name();
TreeNode<Set<String>> hostComponentNode = resultTree.getChild(
hostType + "/" + hostComponentType);

if (hostComponentNode == null) {
TreeNode<Set<String>> hostNode = resultTree.getChild(hostType);
if (hostNode == null) {
hostNode = resultTree.addChild(new HashSet<>(), hostType);
}
hostComponentNode = hostNode.addChild(new HashSet<>(), hostComponentType);
}
resultTree.getChild(configType).getObject().add("properties");
hostComponentNode.getObject().add("HostRoles/component_name");
ensureChild(resultTree, Resource.Type.ClusterSetting.name());

TreeNode<Set<String>> serviceGroupNode = ensureChild(resultTree, Resource.Type.ServiceGroup.name());
TreeNode<Set<String>> serviceNode = ensureChild(serviceGroupNode, Resource.Type.Service.name());
ensureChild(serviceNode, Resource.Type.Component.name(),
"ServiceComponentInfo/cluster_name",
"ServiceComponentInfo/service_name",
"ServiceComponentInfo/component_name",
"ServiceComponentInfo/recovery_enabled");

TreeNode<Set<String>> hostNode = ensureChild(resultTree, Resource.Type.Host.name());
ensureChild(hostNode, Resource.Type.HostComponent.name(), "HostRoles/component_name");

return resultTree;
}

private TreeNode<Set<String>> ensureChild(TreeNode<Set<String>> parent,
String resourceType,
String... properties) {
TreeNode<Set<String>> child = parent.getChild(resourceType);
if (null == child) {
child = parent.addChild(new HashSet<>(), resourceType);
}
for (String property: properties) {
child.getObject().add(property);
}
return child;
}


@Override
public Result finalizeResult(Result queryResult) {
TreeNode<Resource> resultTree = queryResult.getResultTree();
Expand Down Expand Up @@ -198,11 +199,10 @@ private Resource createBlueprintResource(TreeNode<Resource> clusterNode) {
configProcessor.doUpdateForBlueprintExport();

Set<StackId> stackIds = topology.getBlueprint().getStackIds();
if (stackIds.size() == 1) {
StackId stackId = Iterables.getOnlyElement(stackIds);
blueprintResource.setProperty("Blueprints/stack_name", stackId.getStackName());
blueprintResource.setProperty("Blueprints/stack_version", stackId.getStackVersion());
}
// TODO: mpacks should come from service groups once https://github.com/apache/ambari/pull/234 will be committed
Collection<Map<String, String>> mpackInstances = stackIds.stream().

@rnettleton rnettleton Feb 28, 2018

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will this query work for clusters that have been created by the Ambari UI?

@benyoka benyoka Mar 1, 2018

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I am testing it with a cluster that has been created via UI.

map( stackId -> ImmutableMap.of("name", stackId.getStackName(), "version", stackId.getStackVersion())).collect(toList());
blueprintResource.setProperty(BlueprintResourceProvider.MPACK_INSTANCES_PROPERTY_ID, mpackInstances);

if (topology.isClusterKerberosEnabled()) {
Map<String, Object> securityConfigMap = new LinkedHashMap<>();
Expand Down Expand Up @@ -238,7 +238,7 @@ private Resource createBlueprintResource(TreeNode<Resource> clusterNode) {
/***
* Constructs the Settings object of the following form:
* "settings": [ {
"recovery_settings": [
"cluster_settings": [
{
"recovery_enabled": "true"
} ] },
Expand All @@ -262,6 +262,8 @@ private Resource createBlueprintResource(TreeNode<Resource> clusterNode) {
"recovery_enabled": "true"
} ] } ]
*
* <b>NOTE:</b> As of 3.0 global recovery settings will move under a new section called cluster_settings.
*
* @param clusterNode
* @return A Collection<Map<String, Object>> which represents the Setting Object
*/
Expand All @@ -271,25 +273,27 @@ private Collection<Map<String, Object>> getSettings(TreeNode<Resource> clusterNo
//Initialize collections to create appropriate json structure
Collection<Map<String, Object>> blueprintSetting = new ArrayList<>();

Set<Map<String, String>> recoverySettingValue = new HashSet<>();
Set<Map<String, String>> serviceSettingValue = new HashSet<>();
Set<Map<String, String>> componentSettingValue = new HashSet<>();

HashMap<String, String> property = new HashMap<>();
HashMap<String, String> componentProperty = new HashMap<>();
Boolean globalRecoveryEnabled = false;

//Fetch the services, to obtain ServiceInfo and ServiceComponents
Collection<TreeNode<Resource>> serviceChildren = clusterNode.getChild("services").getChildren();
// TODO: set mpack instance if needed (multi-mpack case)
Collection<TreeNode<Resource>> serviceChildren =
clusterNode.getChild("servicegroups").
getChildren().stream().flatMap(
node -> node.getChild("services").getChildren().stream()).
collect(toList());

HashMap<String, String> serviceProperty;
for (TreeNode serviceNode : serviceChildren) {
ResourceImpl service = (ResourceImpl) serviceNode.getObject();
Map<String, Object> ServiceInfoMap = service.getPropertiesMap().get("ServiceInfo");

//service_settings population
property = new HashMap<>();
serviceProperty = new HashMap<>();
if (ServiceInfoMap.get("credential_store_enabled").equals("true")) {
property.put("name", ServiceInfoMap.get("service_name").toString());
property.put("credential_store_enabled", "true");
serviceProperty.put("name", ServiceInfoMap.get("service_name").toString());
serviceProperty.put("credential_store_enabled", "true");
}

//Fetch the service Components to obtain ServiceComponentInfo
Expand All @@ -299,34 +303,36 @@ private Collection<Map<String, Object>> getSettings(TreeNode<Resource> clusterNo
Map<String, Object> ServiceComponentInfoMap = component.getPropertiesMap().get("ServiceComponentInfo");

if (ServiceComponentInfoMap.get("recovery_enabled").equals("true")) {
globalRecoveryEnabled = true;
property.put("name", ServiceInfoMap.get("service_name").toString());
property.put("recovery_enabled", "true");
serviceProperty.put("name", ServiceInfoMap.get("service_name").toString());
serviceProperty.put("recovery_enabled", "true");

//component_settings population
componentProperty = new HashMap<>();
HashMap<String, String> componentProperty = new HashMap<>();
componentProperty.put("name", ServiceComponentInfoMap.get("component_name").toString());
componentProperty.put("recovery_enabled", "true");
componentSettingValue.add(componentProperty);
}
}

if (!property.isEmpty())
serviceSettingValue.add(property);
if (!componentProperty.isEmpty())
componentSettingValue.add(componentProperty);
if (!serviceProperty.isEmpty())
serviceSettingValue.add(serviceProperty);
}
//recovery_settings population
property = new HashMap<>();
if (globalRecoveryEnabled) {
property.put("recovery_enabled", "true");
} else {
property.put("recovery_enabled", "false");

// Add cluster settings
Set<Map<String, String>> clusterSettings = new HashSet<>();
TreeNode<Resource> settingsNode = clusterNode.getChild("settings");
if (null != settingsNode) {
for (TreeNode<Resource> clusterSettingNode: settingsNode.getChildren()) {
Map<String, Object> nodeProperties = clusterSettingNode.getObject().getPropertiesMap().get(ClusterSettingResourceProvider.RESPONSE_KEY);
String key = Objects.toString(nodeProperties.get(ClusterSettingResourceProvider.CLUSTER_SETTING_NAME_PROPERTY_ID));
String value = Objects.toString(nodeProperties.get(ClusterSettingResourceProvider.CLUSTER_SETTING_VALUE_PROPERTY_ID));
clusterSettings.add(ImmutableMap.of(key, value));
}
}
recoverySettingValue.add(property);

//Add all the different setting values.
Map<String, Object> settingMap = new HashMap<>();
settingMap.put("recovery_settings", recoverySettingValue);
settingMap.put("cluster_settings", clusterSettings);
blueprintSetting.add(settingMap);

settingMap = new HashMap<>();
Expand Down
2 changes: 1 addition & 1 deletion ...in/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -522,7 +522,7 @@ public void doUpdateForBlueprintExport() {

doFilterPriorToExport(configuration);
}
}
}

@adoroszlai adoroszlai Feb 22, 2018

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems to be accidental.


/**
* This method iterates over the properties passed in, and applies a
Expand Down
7 changes: 5 additions & 2 deletions ...ain/java/org/apache/ambari/server/controller/internal/ClusterSettingResourceProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,13 +70,16 @@ public class ClusterSettingResourceProvider extends AbstractControllerResourcePr

// ----- Property ID constants ---------------------------------------------

public static final String CLUSTER_SETTING_NAME_PROPERTY_ID = "cluster_setting_name";
public static final String CLUSTER_SETTING_VALUE_PROPERTY_ID = "cluster_setting_value";

public static final String RESPONSE_KEY = "ClusterSettingInfo";
public static final String ALL_PROPERTIES = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "*";
public static final String CLUSTER_SETTING_CLUSTER_ID_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "cluster_id";
public static final String CLUSTER_SETTING_CLUSTER_NAME_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "cluster_name";
public static final String CLUSTER_SETTING_CLUSTER_SETTING_ID_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "cluster_setting_id";
public static final String CLUSTER_SETTING_CLUSTER_SETTING_NAME_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "cluster_setting_name";
public static final String CLUSTER_SETTING_CLUSTER_SETTING_VALUE_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "cluster_setting_value";
public static final String CLUSTER_SETTING_CLUSTER_SETTING_NAME_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + CLUSTER_SETTING_NAME_PROPERTY_ID;
public static final String CLUSTER_SETTING_CLUSTER_SETTING_VALUE_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + CLUSTER_SETTING_VALUE_PROPERTY_ID;

private static final Set<String> pkPropertyIds =
Sets.newHashSet(new String[]{
Expand Down
9 changes: 4 additions & 5 deletions ...er/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,14 +154,13 @@ private void createHostGroupInfo(Collection<ExportedHostGroup> exportedHostGroup


private Stack parseStack(Resource clusterResource) throws InvalidTopologyTemplateException {
String[] stackTokens = String.valueOf(clusterResource.getPropertyValue(
ClusterResourceProvider.CLUSTER_VERSION_PROPERTY_ID)).split("-");

StackId stackId = new StackId(clusterResource.getPropertyValue(
ClusterResourceProvider.CLUSTER_VERSION_PROPERTY_ID).toString());
try {
return new Stack(stackTokens[0], stackTokens[1], controller);
return new Stack(stackId.getStackName(), stackId.getStackVersion(), controller);
} catch (AmbariException e) {
throw new InvalidTopologyTemplateException(String.format(
"The specified stack doesn't exist: name=%s version=%s", stackTokens[0], stackTokens[1]));
"The specified stack doesn't exist: name=%s version=%s", stackId.getStackName(), stackId.getStackVersion()));
}
}

Expand Down
Loading