Merge branch-feature-ambari-14714 into branch-feature-ambari-14714-ui

ambari-common/src/main/python/resource_management/libraries/functions/config_helper.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+
+def get_mpack_name(config):
+  return config['hostLevelParams']['stack_name']
+
+
+def get_mpack_version(config):
+  return config['hostLevelParams']['stack_version']
+
+
+def get_mpack_instance_name(config):
+  return config['serviceGroupName']
+
+
+def get_module_name(config):
+  return config['serviceName']
+
+
+def get_component_type(config):
+  return config['role']
+
+
+def get_component_instance_name(config):
+  return "default"

ambari-common/src/main/python/resource_management/libraries/functions/get_not_managed_resources.py

@@ -25,6 +25,7 @@
 from resource_management.libraries.script import Script
 from resource_management.core.logger import Logger
 from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.cluster_settings import get_cluster_setting_value, get_cluster_setting
 
 def get_not_managed_resources():
   """
@@ -34,8 +35,8 @@ def get_not_managed_resources():
   """
   config = Script.get_config()
   not_managed_hdfs_path_list = json.loads(config['hostLevelParams']['not_managed_hdfs_path_list'])[:]
-  if 'managed_hdfs_resource_property_names' in config['configurations']['cluster-env']:
-    managed_hdfs_resource_property_names = config['configurations']['cluster-env']['managed_hdfs_resource_property_names']
+  if get_cluster_setting('managed_hdfs_resource_property_names') is not None:
+    managed_hdfs_resource_property_names = get_cluster_setting_value('managed_hdfs_resource_property_names')
     managed_hdfs_resource_property_list = filter(None, [property.strip() for property in managed_hdfs_resource_property_names.split(',')])
 
     for property_name in managed_hdfs_resource_property_list:

ambari-common/src/main/python/resource_management/libraries/functions/mpack_manager_helper.py

@@ -0,0 +1,91 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+import os
+from instance_manager import create_mpack, set_mpack_instance, get_conf_dir, list_instances
+
+CONFIG_DIR_KEY_NAME = 'config_dir'
+PATH_KEY_NAME = 'path'
+COMPONENTS_PLURAL_KEY_NAME = 'components'
+COMPONENT_INSTANCES_PLURAL_KEY_NAME = 'component-instances'
+
+
+def get_component_conf_path(mpack_name, instance_name, module_name, components_instance_type,
+                            subgroup_name='default', component_instance_name='default'):
+  """
+  :returns the single string that contains the path to the configuration folder of given component instance
+  :raises ValueError if the parameters doesn't match the mpack or instances structure
+  """
+
+  conf_json = get_conf_dir(mpack_name, instance_name, subgroup_name, module_name,
+                           {components_instance_type: [component_instance_name]})
+
+  return conf_json[COMPONENTS_PLURAL_KEY_NAME][components_instance_type.lower()][COMPONENT_INSTANCES_PLURAL_KEY_NAME][
+    component_instance_name][CONFIG_DIR_KEY_NAME]
+
+
+def get_component_target_path(mpack_name, instance_name, module_name, components_instance_type,
+                              subgroup_name='default', component_instance_name='default'):
+  """
+  :returns the single string that contains the path to the mpack component folder of given component instance
+  :raises ValueError if the parameters doesn't match the mpack or instances structure
+  """
+
+  instances_json = list_instances(mpack_name, instance_name, subgroup_name, module_name,
+                                  {components_instance_type: [component_instance_name]})
+
+  return instances_json[COMPONENTS_PLURAL_KEY_NAME][components_instance_type.lower()][
+    COMPONENT_INSTANCES_PLURAL_KEY_NAME][component_instance_name][PATH_KEY_NAME]
+
+
+def get_component_home_path(mpack_name, instance_name, module_name, components_instance_type,
+                            subgroup_name='default', component_instance_name='default'):
+  """
+  :returns the single string that contains the path to the module component folder of given component instance
+  :raises ValueError if the parameters doesn't match the mpack or instances structure
+  """
+
+  component_path = get_component_target_path(mpack_name=mpack_name, instance_name=instance_name,
+                                             subgroup_name=subgroup_name,
+                                             module_name=module_name, components_instance_type=components_instance_type,
+                                             component_instance_name=component_instance_name)
+
+  return os.readlink(component_path)
+
+
+def create_component_instance(mpack_name, mpack_version, instance_name, module_name, components_instance_type,
+                              subgroup_name='default', component_instance_name='default'):
+  """
+  creates the single component instance according to the parameters
+  :raises ValueError if the parameters doesn't match the mpack or instances structure
+  """
+  create_mpack(mpack_name, mpack_version, instance_name, subgroup_name, module_name,
+               None, {components_instance_type: [component_instance_name]})
+
+
+def set_component_instance_version(mpack_name, mpack_version, instance_name, module_name, components_instance_type,
+                                   subgroup_name='default', component_instance_name='default'):
+  """
+  changes the version of the single component instance according to the parameters
+  :raises ValueError if the parameters doesn't match the mpack or instances structure
+  """
+  set_mpack_instance(mpack_name, mpack_version, instance_name, subgroup_name, module_name,
+                     None, {components_instance_type: [component_instance_name]})

ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py

@@ -51,13 +51,13 @@ def create_repo_files(self):
     Creates repositories in a consistent manner for all types
     :return: a dictionary with repo ID => repo file name mapping
     """
-    if self.command_repository.version_id is None:
+    if self.command_repository.mpack_id is None:
       raise Fail("The command repository was not parsed correctly")
 
     if 0 == len(self.command_repository.items):
       Logger.warning(
         "Repository for {0}/{1} has no repositories.  Ambari may not be managing this version.".format(
-          self.command_repository.stack_name, self.command_repository.version_string))
+          self.command_repository.mpack_name, self.command_repository.version_string))
       return {}
 
     append_to_file = False  # initialize to False to create the file anew.
@@ -73,7 +73,7 @@ def create_repo_files(self):
       if not repository.ambari_managed:
         Logger.warning(
           "Repository for {0}/{1}/{2} is not managed by Ambari".format(
-            self.command_repository.stack_name, self.command_repository.version_string, repository.repo_id))
+            self.command_repository.mpack_name, self.command_repository.version_string, repository.repo_id))
       else:
         Repository(repository.repo_id,
                    action="create",
@@ -133,10 +133,9 @@ def __init__(self, repo_object):
     else:
       raise Fail("Cannot deserialize command repository {0}".format(str(repo_object)))
 
-    # version_id is the primary id of the repo_version table in the database
-    self.version_id = _find_value(json_dict, 'repoVersionId')
-    self.stack_name = _find_value(json_dict, 'stackName')
-    self.version_string = _find_value(json_dict, 'repoVersion')
+    self.mpack_id = _find_value(json_dict, 'mpackId')
+    self.mpack_name = _find_value(json_dict, 'mpackName')
+    self.version_string = _find_value(json_dict, 'mpackVersion')
     self.repo_filename = _find_value(json_dict, 'repoFileName')
     self.feat = CommandRepositoryFeature(_find_value(json_dict, "feature", default={}))
     self.items = []

ambari-common/src/main/python/resource_management/libraries/script/script.py

@@ -61,6 +61,8 @@
 from resource_management.libraries.functions.show_logs import show_logs
 from resource_management.core.providers import get_provider
 from resource_management.libraries.functions.fcntl_based_process_lock import FcntlBasedProcessLock
+from resource_management.libraries.functions.config_helper import get_mpack_name, get_mpack_version, \
+  get_mpack_instance_name, get_module_name, get_component_type, get_component_instance_name
 
 import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
 
@@ -791,6 +793,21 @@ def load_available_packages(self):
       Logger.exception("Unable to load available packages")
       self.available_packages_in_repos = []
 
+  def create_component_instance(self):
+    # should be used only when mpack-instance-manager is available
+    from resource_management.libraries.functions.mpack_manager_helper import create_component_instance
+    config = self.get_config()
+    mpack_name = get_mpack_name(config)
+    mpack_version = get_mpack_version(config)
+    mpack_instance_name = get_mpack_instance_name(config)
+    module_name = get_module_name(config)
+    component_type = get_component_type(config)
+    component_instance_name = get_component_instance_name(config)
+
+    create_component_instance(mpack_name=mpack_name, mpack_version=mpack_version, instance_name=mpack_instance_name,
+                              module_name=module_name, components_instance_type=component_type,
+                              component_instance_name=component_instance_name)
+
 
   def install_packages(self, env):
     """

ambari-project/pom.xml

@@ -485,6 +485,13 @@
         <artifactId>swagger-maven-plugin</artifactId>
         <version>${swagger.maven.plugin.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.inferred</groupId>
+        <artifactId>freebuilder</artifactId>
+        <version>1.14.8</version>
+        <optional>true</optional>
+        <scope>provided</scope>
+      </dependency>
     </dependencies>
   </dependencyManagement>
   <build>
@@ -508,7 +515,10 @@
             <encoding>UTF-8</encoding>
             <consoleOutput>true</consoleOutput>
             <failsOnError>true</failsOnError>
-            <includeTestSourceDirectory>true</includeTestSourceDirectory>
+            <sourceDirectories>
+              <sourceDirectory>src/main/java</sourceDirectory>
+              <sourceDirectory>src/test/java</sourceDirectory>
+            </sourceDirectories>
             <linkXRef>false</linkXRef>
           </configuration>
           <executions>

ambari-server/pom.xml

@@ -1644,6 +1644,10 @@
       <version>0.1.10</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.inferred</groupId>
+      <artifactId>freebuilder</artifactId>
+    </dependency>
   </dependencies>
 
   <pluginRepositories>

ambari-server/src/main/java/org/apache/ambari/annotations/ExperimentalFeature.java

@@ -51,5 +51,11 @@ public enum ExperimentalFeature {
    * For code that is for upgrading Mpacks.  Use this to mark code that may ultimately
    * be removed.
    */
-  MPACK_UPGRADES
+  MPACK_UPGRADES,
+
+  /**
+   * Used to mark code that is required for successful removal and refactoring
+   * when repo versions are dropped.
+   */
+  REPO_VERSION_REMOVAL;
 }

ambari-server/src/main/java/org/apache/ambari/server/ServiceComponentHostNotFoundException.java

@@ -31,4 +31,13 @@ public ServiceComponentHostNotFoundException(String clusterName,
         + ", hostName=" + hostName);
   }
 
+  public ServiceComponentHostNotFoundException(String clusterName,
+      String serviceName, Long serviceComponentId, String hostName) {
+    super("ServiceComponentHost not found"
+            + ", clusterName=" + clusterName
+            + ", serviceName=" + serviceName
+            + ", serviceComponentId=" + serviceComponentId
+            + ", hostName=" + hostName);
+  }
+
 }

ambari-server/src/main/java/org/apache/ambari/server/StateRecoveryManager.java

@@ -20,10 +20,10 @@
 
 import java.util.List;
 
-import org.apache.ambari.server.orm.dao.HostVersionDAO;
-import org.apache.ambari.server.orm.dao.ServiceComponentDesiredStateDAO;
-import org.apache.ambari.server.orm.entities.HostVersionEntity;
-import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.orm.dao.MpackHostStateDAO;
+import org.apache.ambari.server.orm.entities.MpackHostStateEntity;
+import org.apache.ambari.server.state.Mpack;
 import org.apache.ambari.server.state.RepositoryVersionState;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -39,48 +39,41 @@ public class StateRecoveryManager {
   private static final Logger LOG = LoggerFactory.getLogger(StateRecoveryManager.class);
 
   @Inject
-  private HostVersionDAO hostVersionDAO;
+  private MpackHostStateDAO mpackHostStateDAO;
 
+  /**
+   * Used for looking up {@link Mpack} instances by IDs.
+   */
   @Inject
-  private ServiceComponentDesiredStateDAO serviceComponentDAO;
+  private AmbariMetaInfo ambariMetaInfo;
 
   public void doWork() {
-    checkHostAndClusterVersions();
+    updateManagementPackInstallationState();
   }
 
-  void checkHostAndClusterVersions() {
-    List<HostVersionEntity> hostVersions = hostVersionDAO.findAll();
-    for (HostVersionEntity hostVersion : hostVersions) {
-      if (hostVersion.getState().equals(RepositoryVersionState.INSTALLING)) {
-        hostVersion.setState(RepositoryVersionState.INSTALL_FAILED);
+  /**
+   * Resets any management pack installation states from
+   * {@link RepositoryVersionState#INSTALLING} to
+   * {@link RepositoryVersionState#INSTALL_FAILED}.
+   */
+  void updateManagementPackInstallationState() {
+    List<MpackHostStateEntity> mpackHostStates = mpackHostStateDAO.findAll();
+    for (MpackHostStateEntity mpackHostState : mpackHostStates) {
+      if (mpackHostState.getState() == RepositoryVersionState.INSTALLING) {
+        mpackHostState.setState(RepositoryVersionState.INSTALL_FAILED);
+
+        Mpack mpack = ambariMetaInfo.getMpack(mpackHostState.getMpackId());
+
         String msg = String.format(
-                "Recovered state of host version %s on host %s from %s to %s",
-                hostVersion.getRepositoryVersion().getDisplayName(),
-                hostVersion.getHostName(),
+                "The installation state of management pack %s on host %s was set from %s to %s",
+            mpack.getName(),
+                mpackHostState.getHostName(),
                 RepositoryVersionState.INSTALLING,
                 RepositoryVersionState.INSTALL_FAILED);
         LOG.warn(msg);
-        hostVersionDAO.merge(hostVersion);
-      }
-    }
 
-    List<ServiceComponentDesiredStateEntity> components = serviceComponentDAO.findAll();
-    for (ServiceComponentDesiredStateEntity component : components) {
-      if (RepositoryVersionState.INSTALLING == component.getRepositoryState()) {
-        component.setRepositoryState(RepositoryVersionState.INSTALL_FAILED);
-        serviceComponentDAO.merge(component);
-        String msg = String.format(
-            "Recovered state of cluster %s of component %s/%s for version %s from %s to %s",
-            component.getClusterId(),
-            component.getServiceId(),
-            component.getComponentName(),
-            component.getDesiredRepositoryVersion().getDisplayName(),
-            RepositoryVersionState.INSTALLING,
-            RepositoryVersionState.INSTALL_FAILED);
-        LOG.warn(msg);
+        mpackHostStateDAO.merge(mpackHostState);
       }
     }
   }
-
-
 }