Skip to content

[AMBARI-23121] Fix few instance manager issues found during ZK deploment (dsen) #518

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
d0zen1 merged 2 commits into from
Mar 2, 2018
Merged

[AMBARI-23121] Fix few instance manager issues found during ZK deploment (dsen) #518

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
26a98c9
[AMBARI-23121] Fix few instance manager issues found during ZK deploy…
d0zen1 Mar 1, 2018
166e3cc
[AMBARI-23121] Fix few instance manager issues found during ZK deploy…
d0zen1 Mar 2, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 32 additions & 13 deletions mpack-instance-manager/src/main/python/instance_manager/instance_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ def create_mpack(mpack_name, mpack_version, mpack_instance, subgroup_name=DEFAUL
OR
list of 'components_instances_name' to be created. (default) OR '*' for all components
"""
mpack_name = mpack_name.lower()
module_name = module_name.lower()
mpack_name, module_name, components, components_map = normalize_parameters(
mpack_name, module_name, components, components_map)

validate_mpack_for_creation_or_changing(mpack_name, mpack_version, module_name, components, components_map)

Expand All @@ -71,14 +71,17 @@ def set_mpack_instance(mpack, mpack_version, mpack_instance, subgroup_name=DEFAU
OR
list of 'components_instances_name' to be created. (default) OR '*' for all components
"""
mpack = mpack.lower()
module_name = module_name.lower()
mpack, module_name, components, components_map = normalize_parameters(
mpack, module_name, components, components_map)

instances = MpackInstance.parse_instances_with_filtering(os.path.join(ROOT_FOLDER_PATH, INSTANCES_FOLDER_NAME), mpack,
mpack_instance, subgroup_name, module_name,
components, components_map)
if not instances:
raise ValueError("Found no instances for the given filters.")
raise ValueError("Found no instances for the given filters: mpack_name:{0}, instance_name:{1},"
" subgroup_name:{2}, module_name:{3}, components:{4}, components_map:{5}".
format(mpack, mpack_instance, subgroup_name, module_name,
components, components_map))

validate_mpack_for_creation_or_changing(mpack, mpack_version, module_name, components, components_map)

Expand Down Expand Up @@ -142,11 +145,8 @@ def build_granular_json_with_filtering(mpack_name_filter, instance_name_filter,

The output_conf_dir or output_path for each component instance will be included in json depending on given parameters.
"""

if mpack_name_filter:
mpack_name_filter = mpack_name_filter.lower()
if module_name_filter:
module_name_filter = module_name_filter.lower()
mpack_name_filter, module_name_filter, skipped, components_name_filter_map = normalize_parameters(
mpack_name_filter, module_name_filter, [], components_name_filter_map)

instances = MpackInstance.parse_instances_with_filtering(os.path.join(ROOT_FOLDER_PATH, INSTANCES_FOLDER_NAME),
mpack_name_filter,
Expand All @@ -155,7 +155,10 @@ def build_granular_json_with_filtering(mpack_name_filter, instance_name_filter,
None,
components_name_filter_map)
if not instances:
raise ValueError("Found no instances for the given filters.")
raise ValueError("Found no instances for the given filters: mpack_name:{0}, instance_name:{1},"
" subgroup_name:{2}, module_name:{3}, components_name_map:{4}".
format(mpack_name_filter, instance_name_filter, subgroup_name_filter,
module_name_filter, components_name_filter_map))

full_json_output = build_json_output(instances, output_conf_dir=output_conf_dir, output_path=output_path)

Expand Down Expand Up @@ -229,6 +232,22 @@ def build_json_output(instances, output_conf_dir=False, output_path=False):
return {'mpacks': result}


def normalize_parameters(mpack_name, module_name, components, components_map):

@adoroszlai adoroszlai Mar 2, 2018

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks!

"""
normalize parameters to be lowercase, for components_map dictionary only keys are normalized
"""
if mpack_name:
mpack_name = mpack_name.lower()
if module_name:
module_name = module_name.lower()
if components and components != "*":
components = [component.lower() for component in components]
if components_map:
components_map = dict((k.lower(), v) for k, v in components_map.iteritems())

return mpack_name, module_name, components, components_map


def validate_mpack_for_creation_or_changing(mpack_name, mpack_version, module_name, components, components_map):
mpack_root_path = os.path.join(ROOT_FOLDER_PATH, MPACKS_FOLDER_NAME, mpack_name)
if not os.path.exists(mpack_root_path):
Expand Down Expand Up @@ -478,7 +497,7 @@ def parse_into_module_instance_dict(path, module_name_filter, components_filter,
(components_name_filter_map and folder_name in components_name_filter_map)):
components_map = ComponentInstance(name=DEFAULT_COMPONENT_INSTANCE_NAME,
component_path=os.path.join(path, folder_name),
path_exec=os.path.realpath(
path_exec=os.readlink(
os.path.join(path, folder_name, CURRENT_SOFTLINK_NAME)))
else:
components_map = ComponentInstance.parse_into_components_dict(os.path.join(path, folder_name),
Expand Down Expand Up @@ -566,7 +585,7 @@ def parse_into_component_instance_dict(path, component_names_filter=None):
if not component_names_filter or component_instance_name in component_names_filter:
result[component_instance_name] = ComponentInstance(name=component_instance_name,
component_path=os.path.join(path, component_instance_name),
path_exec=os.path.realpath(
path_exec=os.readlink(
os.path.join(path, component_instance_name,
CURRENT_SOFTLINK_NAME)))
return result
Expand Down
17 changes: 13 additions & 4 deletions mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,14 @@ def check_required_options(options, action, parser):
if not options.module_name:
missing_options.append("module-name")

if missing_options:
parser.error("Missing following required command options: {0}".format(missing_options))
if missing_options:
parser.error("Missing following required command options: {0}".format(missing_options))

if not options.components_map and not options.components:
parser.error("Either components or components-map option must be specified.")

if options.components_map and options.components:
parser.error("Only components or components-map option could be specified. Can't use both.")


def init_create_parser_options(parser):
Expand Down Expand Up @@ -117,8 +123,11 @@ def main(options, args):
parsed_components = None
parsed_components_map = None
try:
if hasattr(options, 'components') and options.components and options.components != '*':
parsed_components = ast.literal_eval(options.components)
if hasattr(options, 'components') and options.components:
if options.components == '*':
parsed_components = '*'
else:
parsed_components = ast.literal_eval(options.components)
if options.components_map:
parsed_components_map = ast.literal_eval(options.components_map)
except ValueError:
Expand Down
43 changes: 29 additions & 14 deletions mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ def test_create_mpack_server_module_with_default_component_instance(self):
instance_manager.DEFAULT_MPACK_INSTANCE_NAME)))

def test_create_mpack_server_module_with_multiple_component_instances(self):
create_mpack_with_defaults(components=None, components_map={SERVER_COMPONENT_NAME: ['server1', 'server2']})
create_mpack_with_defaults(components=None, components_map={SERVER_COMPONENT_NAME.upper(): ['server1', 'server2']})

current_link_1 = os.path.join(TMP_ROOT_FOLDER, instance_manager.INSTANCES_FOLDER_NAME, MPACK_NAME,
INSTANCE_NAME_1, SUBGROUP_NAME, SERVER_MODULE_NAME, SERVER_COMPONENT_NAME,
Expand Down Expand Up @@ -121,12 +121,12 @@ def test_set_version_server_module_asterisk(self):
MPACK_VERSION_2, SERVER_COMPONENT_NAME))

def test_set_version_client_module_asterisk(self):
create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME)
create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME.upper())

build_rpm_structure(mpack_version=MPACK_VERSION_2, remove_old_content=False, create_modules=False)

instance_manager.set_mpack_instance(MPACK_NAME, MPACK_VERSION_2, INSTANCE_NAME_1, SUBGROUP_NAME,
CLIENT_MODULE_NAME, '*')
instance_manager.set_mpack_instance(MPACK_NAME.upper(), MPACK_VERSION_2, INSTANCE_NAME_1, SUBGROUP_NAME,
CLIENT_MODULE_NAME.upper(), '*')

current_link = os.path.join(TMP_ROOT_FOLDER, instance_manager.INSTANCES_FOLDER_NAME, MPACK_NAME,
INSTANCE_NAME_1, SUBGROUP_NAME, CLIENT_COMPONENT_NAME,
Expand Down Expand Up @@ -160,9 +160,9 @@ def test_set_version_for_one_of_two_component_instances(self):
MPACK_VERSION_2, SERVER_COMPONENT_NAME))

def test_get_conf_dir_all(self):
create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME)
create_mpack_with_defaults(module_name=SERVER_MODULE_NAME, components=None,
components_map={SERVER_COMPONENT_NAME: ['server1']})
create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME.upper())
create_mpack_with_defaults(module_name=SERVER_MODULE_NAME.upper(), components=None,
components_map={SERVER_COMPONENT_NAME.upper(): ['server1']})

conf_dir_json = instance_manager.get_conf_dir()

Expand Down Expand Up @@ -210,9 +210,9 @@ def test_get_conf_dir_all(self):
self.assertEqual(conf_dir_json, expected_json)

def test_list_instances_all(self):
create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME)
create_mpack_with_defaults(module_name=SERVER_MODULE_NAME, components=None,
components_map={SERVER_COMPONENT_NAME: ['server1']})
create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME.upper())
create_mpack_with_defaults(module_name=SERVER_MODULE_NAME.upper(), components=None,
components_map={SERVER_COMPONENT_NAME.upper(): ['server1']})

conf_dir_json = instance_manager.list_instances()

Expand All @@ -232,7 +232,7 @@ def test_list_instances_all(self):
"hdfs_server": {
"component-instances": {
"server1": {
"path": "/tmp/instance_manager_test/modules/hdfs/3.1.0.0-b1",
"path": "/tmp/instance_manager_test/mpacks/hdpcore/1.0.0-b1/hdfs_server",
"name": "server1"
}
}
Expand All @@ -243,7 +243,7 @@ def test_list_instances_all(self):
"category": "CLIENT",
"component_instances": {
"default": {
"path": "/tmp/instance_manager_test/modules/hdfs-clients/3.1.0.0-b1",
"path": "/tmp/instance_manager_test/mpacks/hdpcore/1.0.0-b1/hdfs_client",
"name": "default"
}
},
Expand Down Expand Up @@ -317,7 +317,7 @@ def test_filtering(self):
expected_filter_result = {'mpacks': {'edw': {'mpack-instances': {'eCommerce': {'name': 'eCommerce', 'subgroups': {
'default': {'modules': {'hdfs': {'category': 'SERVER', 'name': 'hdfs', 'components': {'hdfs_server': {
'component-instances': {
'server2': {'path': '/tmp/instance_manager_test/modules/hdfs/3.1.0.0-b1', 'name': 'server2'}}}}}}}}}}}}}
'server2': {'path': '/tmp/instance_manager_test/mpacks/edw/1.0.0-b1/hdfs_server', 'name': 'server2'}}}}}}}}}}}}}
self.assertEquals(expected_filter_result, filter_by_component_instance_name_json)

def test_validation(self):
Expand Down Expand Up @@ -363,6 +363,19 @@ def test_creating_existing_component_instance(self):
"The instance /tmp/instance_manager_test/instances/hdpcore/Production/default/hdfs/"
"hdfs_server/default already exist. To change the version use set-mpack-instance command")

def test_normalize_parameters(self):
mpack_name = MPACK_NAME.upper()
module_name = SERVER_MODULE_NAME.upper()
components = [SERVER_COMPONENT_NAME.upper()]
components_map = {SERVER_COMPONENT_NAME.upper(): ["DEFAULT"]}
mpack_name, module_name, components, components_map = instance_manager.normalize_parameters(
mpack_name, module_name, components, components_map)

self.assertEquals(mpack_name, MPACK_NAME.lower())
self.assertEquals(module_name, SERVER_MODULE_NAME.lower())
self.assertEquals(components, [SERVER_COMPONENT_NAME.lower()])
self.assertEquals(components_map, {SERVER_COMPONENT_NAME.lower(): ["DEFAULT"]})

def test_set_non_existing_instance(self):
try:
instance_manager.set_mpack_instance(mpack=MPACK_NAME, mpack_version=MPACK_VERSION_1,
Expand All @@ -383,7 +396,9 @@ def test_set_non_existing_instance(self):
raise AssertionError("The previous call should have thrown exception")
except ValueError as e:
self.assertEquals(e.message,
"Found no instances for the given filters.")
"Found no instances for the given filters: mpack_name:hdpcore, instance_name:Production,"
" subgroup_name:default, module_name:hdfs, components:None,"
" components_map:{'hdfs_server': ['non-existing-instance']}")


def create_mpack_with_defaults(mpack_name=MPACK_NAME, mpack_version=MPACK_VERSION_1, mpack_instance=INSTANCE_NAME_1,
Expand Down