Skip to content

AMBARI-23107 - Deprecate Repo Versions And Start Tracking Repo OS's F… #515

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jonathan-hurley merged 3 commits into from
Mar 1, 2018
Merged

AMBARI-23107 - Deprecate Repo Versions And Start Tracking Repo OS's F… #515

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7b831a7
AMBARI-23107 - Deprecate Repo Versions And Start Tracking Repo OS's F…
Mar 1, 2018
b6d1e6b
AMBARI-23107 - Decided to add changes to the other SQL files as well
Mar 1, 2018
cdb9175
AMBARI-23107 - Addressed Nate's comments on formatting and equals()/h…
Mar 1, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 6 additions & 7 deletions ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,13 @@ def create_repo_files(self):
Creates repositories in a consistent manner for all types
:return: a dictionary with repo ID => repo file name mapping
"""
if self.command_repository.version_id is None:
if self.command_repository.mpack_id is None:
raise Fail("The command repository was not parsed correctly")

if 0 == len(self.command_repository.items):
Logger.warning(
"Repository for {0}/{1} has no repositories. Ambari may not be managing this version.".format(
self.command_repository.stack_name, self.command_repository.version_string))
self.command_repository.mpack_name, self.command_repository.version_string))
return {}

append_to_file = False # initialize to False to create the file anew.
Expand All @@ -73,7 +73,7 @@ def create_repo_files(self):
if not repository.ambari_managed:
Logger.warning(
"Repository for {0}/{1}/{2} is not managed by Ambari".format(
self.command_repository.stack_name, self.command_repository.version_string, repository.repo_id))
self.command_repository.mpack_name, self.command_repository.version_string, repository.repo_id))
else:
Repository(repository.repo_id,
action="create",
Expand Down Expand Up @@ -133,10 +133,9 @@ def __init__(self, repo_object):
else:
raise Fail("Cannot deserialize command repository {0}".format(str(repo_object)))

# version_id is the primary id of the repo_version table in the database
self.version_id = _find_value(json_dict, 'repoVersionId')
self.stack_name = _find_value(json_dict, 'stackName')
self.version_string = _find_value(json_dict, 'repoVersion')
self.mpack_id = _find_value(json_dict, 'mpackId')
self.mpack_name = _find_value(json_dict, 'mpackName')
self.version_string = _find_value(json_dict, 'mpackVersion')
self.repo_filename = _find_value(json_dict, 'repoFileName')
self.feat = CommandRepositoryFeature(_find_value(json_dict, "feature", default={}))
self.items = []
Expand Down
8 changes: 7 additions & 1 deletion ambari-server/src/main/java/org/apache/ambari/annotations/ExperimentalFeature.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,5 +51,11 @@ public enum ExperimentalFeature {
* For code that is for upgrading Mpacks. Use this to mark code that may ultimately
* be removed.
*/
MPACK_UPGRADES
MPACK_UPGRADES,

/**
* Used to mark code that is required for successful removal and refactoring
* when repo versions are dropped.
*/
REPO_VERSION_REMOVAL;
}
61 changes: 27 additions & 34 deletions ambari-server/src/main/java/org/apache/ambari/server/StateRecoveryManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@

import java.util.List;

import org.apache.ambari.server.orm.dao.HostVersionDAO;
import org.apache.ambari.server.orm.dao.ServiceComponentDesiredStateDAO;
import org.apache.ambari.server.orm.entities.HostVersionEntity;
import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.orm.dao.MpackHostStateDAO;
import org.apache.ambari.server.orm.entities.MpackHostStateEntity;
import org.apache.ambari.server.state.Mpack;
import org.apache.ambari.server.state.RepositoryVersionState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -39,48 +39,41 @@ public class StateRecoveryManager {
private static final Logger LOG = LoggerFactory.getLogger(StateRecoveryManager.class);

@Inject
private HostVersionDAO hostVersionDAO;
private MpackHostStateDAO mpackHostStateDAO;

/**
* Used for looking up {@link Mpack} instances by IDs.
*/
@Inject
private ServiceComponentDesiredStateDAO serviceComponentDAO;
private AmbariMetaInfo ambariMetaInfo;

public void doWork() {
checkHostAndClusterVersions();
updateManagementPackInstallationState();
}

void checkHostAndClusterVersions() {
List<HostVersionEntity> hostVersions = hostVersionDAO.findAll();
for (HostVersionEntity hostVersion : hostVersions) {
if (hostVersion.getState().equals(RepositoryVersionState.INSTALLING)) {
hostVersion.setState(RepositoryVersionState.INSTALL_FAILED);
/**
* Resets any management pack installation states from
* {@link RepositoryVersionState#INSTALLING} to
* {@link RepositoryVersionState#INSTALL_FAILED}.
*/
void updateManagementPackInstallationState() {
List<MpackHostStateEntity> mpackHostStates = mpackHostStateDAO.findAll();
for (MpackHostStateEntity mpackHostState : mpackHostStates) {
if (mpackHostState.getState() == RepositoryVersionState.INSTALLING) {
mpackHostState.setState(RepositoryVersionState.INSTALL_FAILED);
Copy link
Contributor

@ncole ncole Mar 1, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should consider renaming the enum RepositoryVersionState to some other "thing" - InstallState or something similar. Doesn't need to be in this change, just bringing it up. We can use it for multiple things, possibly for mpacks, modules, etc.

Copy link
Member Author

@jonathan-hurley jonathan-hurley Mar 1, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree. I'm trying to keep the number of files changed manageable, so I didn't want to do it as part of this commit.


Mpack mpack = ambariMetaInfo.getMpack(mpackHostState.getMpackId());

String msg = String.format(
"Recovered state of host version %s on host %s from %s to %s",
hostVersion.getRepositoryVersion().getDisplayName(),
hostVersion.getHostName(),
"The installation state of management pack %s on host %s was set from %s to %s",
mpack.getName(),
mpackHostState.getHostName(),
RepositoryVersionState.INSTALLING,
RepositoryVersionState.INSTALL_FAILED);
LOG.warn(msg);
hostVersionDAO.merge(hostVersion);
}
}

List<ServiceComponentDesiredStateEntity> components = serviceComponentDAO.findAll();
for (ServiceComponentDesiredStateEntity component : components) {
if (RepositoryVersionState.INSTALLING == component.getRepositoryState()) {
component.setRepositoryState(RepositoryVersionState.INSTALL_FAILED);
serviceComponentDAO.merge(component);
String msg = String.format(
"Recovered state of cluster %s of component %s/%s for version %s from %s to %s",
component.getClusterId(),
component.getServiceId(),
component.getComponentName(),
component.getDesiredRepositoryVersion().getDisplayName(),
RepositoryVersionState.INSTALLING,
RepositoryVersionState.INSTALL_FAILED);
LOG.warn(msg);
mpackHostStateDAO.merge(mpackHostState);
}
}
}


}
109 changes: 59 additions & 50 deletions ...-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,18 +34,22 @@
import org.apache.ambari.server.agent.ExecutionCommand;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.spi.SystemException;
import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
import org.apache.ambari.server.orm.dao.MpackDAO;
import org.apache.ambari.server.orm.dao.ServiceGroupDAO;
import org.apache.ambari.server.orm.entities.MpackEntity;
import org.apache.ambari.server.orm.entities.RepoOsEntity;
import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.orm.entities.ServiceGroupEntity;
import org.apache.ambari.server.orm.entities.StackEntity;
import org.apache.ambari.server.orm.entities.UpgradeEntity;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.DesiredConfig;
import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.ModuleComponent;
import org.apache.ambari.server.state.Mpack;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.ServiceComponent;
import org.apache.ambari.server.state.ServiceInfo;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.StackInfo;
Expand Down Expand Up @@ -96,6 +100,19 @@ public class ExecutionCommandWrapper {
@Inject
private Configuration configuration;

/**
* Used to get service groups (and from them mpacks) so that we can set the
* right information on the command.
*/
@Inject
private ServiceGroupDAO serviceGroupDAO;

/**
* Used for retrieving mpack entities by their ID.
*/
@Inject
private MpackDAO mpackDAO;

@AssistedInject
public ExecutionCommandWrapper(@Assisted String jsonExecutionCommand) {
this.jsonExecutionCommand = jsonExecutionCommand;
Expand Down Expand Up @@ -124,7 +141,7 @@ public ExecutionCommand getExecutionCommand() {
return executionCommand;
}

if( null == jsonExecutionCommand ){
if (null == jsonExecutionCommand) {
throw new RuntimeException(
"Invalid ExecutionCommandWrapper, both object and string representations are null");
}
Expand Down Expand Up @@ -232,28 +249,18 @@ public ExecutionCommand getExecutionCommand() {

// setting repositoryFile
final Host host = cluster.getHost(executionCommand.getHostname()); // can be null on internal commands
final String serviceName = executionCommand.getServiceName(); // can be null on executing special RU tasks

if (null == executionCommand.getRepositoryFile() && null != host && null != serviceName) {
if (null == executionCommand.getRepositoryFile() && null != host) {
final CommandRepository commandRepository;
final Service service = cluster.getService(serviceName);
final String componentName = executionCommand.getComponentName();

try {
ServiceGroupEntity serviceGroupEntity = serviceGroupDAO.find(clusterId, executionCommand.getServiceGroupName());
long mpackId = serviceGroupEntity.getStack().getMpackId();
Mpack mpack = ambariMetaInfo.getMpack(mpackId);
MpackEntity mpackEntity = mpackDAO.findById(mpackId);

if (null != componentName) {
ServiceComponent serviceComponent = service.getServiceComponent(componentName);
commandRepository = repoVersionHelper.getCommandRepository(null, serviceComponent, host);
} else {
RepositoryVersionEntity repoVersion = service.getDesiredRepositoryVersion();
RepoOsEntity osEntity = repoVersionHelper.getOSEntityForHost(host, repoVersion);
commandRepository = repoVersionHelper.getCommandRepository(repoVersion, osEntity);
}
executionCommand.setRepositoryFile(commandRepository);

} catch (SystemException e) {
throw new RuntimeException(e);
}
RepoOsEntity osEntity = repoVersionHelper.getOSEntityForHost(mpackEntity, host);
commandRepository = repoVersionHelper.getCommandRepository(mpack, osEntity);
executionCommand.setRepositoryFile(commandRepository);
}

} catch (ClusterNotFoundException cnfe) {
Expand All @@ -264,52 +271,53 @@ public ExecutionCommand getExecutionCommand() {
cnfe.getMessage());

return executionCommand;
} catch (AmbariException e) {
} catch (Exception e) {
throw new RuntimeException(e);
}

return executionCommand;
}

public void setVersions(Cluster cluster) {
// set the repository version for the component this command is for -
// always use the current desired version
String serviceGroupName = executionCommand.getServiceGroupName();
String serviceName = executionCommand.getServiceName();
String componentName = executionCommand.getComponentName();

String serviceType = null;

try {
RepositoryVersionEntity repositoryVersion = null;
if (!StringUtils.isEmpty(serviceName)) {
Service service = cluster.getService(serviceName);
if (null != service) {
serviceType = service.getServiceType();
repositoryVersion = service.getDesiredRepositoryVersion();

String componentName = executionCommand.getComponentName();
if (!StringUtils.isEmpty(componentName)) {
ServiceComponent serviceComponent = service.getServiceComponent(componentName);
if (null != serviceComponent) {
repositoryVersion = serviceComponent.getDesiredRepositoryVersion();
}
}
}
Mpack mpack = null;
StackEntity stackEntity = null;

if (StringUtils.isNotBlank(serviceGroupName)) {
ServiceGroupEntity serviceGroupEntity = serviceGroupDAO.find(cluster.getClusterId(), serviceGroupName);
stackEntity = serviceGroupEntity.getStack();
mpack = ambariMetaInfo.getMpack(stackEntity.getMpackId());
}

Map<String, String> commandParams = executionCommand.getCommandParams();
Service service = cluster.getService(serviceGroupName, serviceName);
if (null != service) {
serviceType = service.getServiceType();
}

if (null != repositoryVersion) {
ModuleComponent moduleComponent = null;
Map<String, String> commandParams = executionCommand.getCommandParams();
if (null != mpack && StringUtils.isNotBlank(serviceName) && StringUtils.isNotBlank(componentName)) {
// only set the version if it's not set and this is NOT an install
// command
// Some stack scripts use version for path purposes. Sending unresolved version first (for
// blueprints) and then resolved one would result in various issues: duplicate directories
// (/hdp/apps/2.6.3.0 + /hdp/apps/2.6.3.0-235), parent directory not found, and file not
// found, etc. Hence requiring repositoryVersion to be resolved.

moduleComponent = mpack.getModuleComponent(serviceName, componentName);
}

if (null != moduleComponent) {
if (!commandParams.containsKey(VERSION)
&& repositoryVersion.isResolved()
&& executionCommand.getRoleCommand() != RoleCommand.INSTALL) {
commandParams.put(VERSION, repositoryVersion.getVersion());
&& executionCommand.getRoleCommand() != RoleCommand.INSTALL) {
commandParams.put(VERSION, moduleComponent.getVersion());
}
}

StackId stackId = repositoryVersion.getStackId();
if (null != stackEntity) {
StackId stackId = new StackId(stackEntity);
StackInfo stackInfo = ambariMetaInfo.getStack(stackId.getStackName(),
stackId.getStackVersion());

Expand All @@ -327,6 +335,7 @@ public void setVersions(Cluster cluster) {
}
}


// set the desired versions of versionable components. This is safe even during an upgrade because
// we are "loading-late": components that have not yet upgraded in an EU will have the correct versions.
executionCommand.setComponentVersions(cluster);
Expand Down
Loading