Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

package org.apache.ambari.server.orm.entities;

import java.util.ArrayList;
import java.util.Collection;

import javax.persistence.Basic;
Expand Down Expand Up @@ -63,17 +64,20 @@ public class BlueprintEntity {
* Unidirectional one-to-one association to {@link StackEntity}
*/
@OneToOne
@JoinColumn(name = "stack_id", unique = false, nullable = false, insertable = true, updatable = false)
@JoinColumn(name = "stack_id", unique = false, nullable = true, insertable = true, updatable = false)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How can stack be null? It's still necessary to specify this when creating a blueprint - even if it's just an mpack association...

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This field is kept only for compatibility purposes. A blueprint will reference mpacks instead of stacks and it can have multiple mpacks.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So if a blueprint references a stack but there are multiple mpacks associated with that stack, how is it determined which to use?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think when mpacks are defined the stack property should be ignored. Another option is to validate that one of the mpack declarations point to the stack.

Do you want to remove the reference to stack?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that leaving the stack without the mpack makes it ambiguous. Do we have problems removing this field? If we're keeping it only for compatibility, then don't we still have to specify it in addition to the mpack?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jonathan-hurley Ambari upgraded from 2.x needs this data at least for blueprint export, possibly other use cases as well. We may later add upgrade logic to move the data to some other table, but as a first cut I think making it optional is OK.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I had thought that there was going to be a conversion to mpacks for HDP and HDF so that after upgrade, there's going to be references to the newly converted mpacks instead of stacks.

private StackEntity stack;

@OneToMany(cascade = CascadeType.ALL, mappedBy = "blueprint")
private Collection<HostGroupEntity> hostGroups;
private Collection<HostGroupEntity> hostGroups = new ArrayList<>();

@OneToMany(cascade = CascadeType.ALL, mappedBy = "blueprint")
private Collection<BlueprintConfigEntity> configurations;
private Collection<BlueprintConfigEntity> configurations = new ArrayList<>();

@OneToMany(cascade = CascadeType.ALL, mappedBy = "blueprint")
private Collection<BlueprintSettingEntity> settings;
private Collection<BlueprintSettingEntity> settings = new ArrayList<>();

@OneToMany(cascade = CascadeType.ALL, mappedBy = "blueprint")
private Collection<BlueprintMpackReferenceEntity> mpackReferences = new ArrayList<>();


/**
Expand Down Expand Up @@ -182,4 +186,12 @@ public String getSecurityDescriptorReference() {
public void setSecurityDescriptorReference(String securityDescriptorReference) {
this.securityDescriptorReference = securityDescriptorReference;
}

public Collection<BlueprintMpackReferenceEntity> getMpackReferences() {
return mpackReferences;
}

public void setMpackReferences(Collection<BlueprintMpackReferenceEntity> mpackReferences) {
this.mpackReferences = mpackReferences;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.ambari.server.orm.entities;

import javax.persistence.Basic;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.Id;
import javax.persistence.IdClass;
import javax.persistence.JoinColumn;
import javax.persistence.Lob;
import javax.persistence.ManyToOne;
import javax.persistence.Table;

@Entity
@Table(name = "blueprint_mpack_configuration")
@IdClass(BlueprintMpackConfigEntityPk.class)
public class BlueprintMpackConfigEntity implements BlueprintConfiguration {

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This class needs documentation.


@Id
@Column(name = "mpack_ref_id", nullable = false, insertable = false, updatable = false)
private Long mpackRefId;

@Id
@Column(name = "type_name", nullable = false, insertable = true, updatable = false, length = 100)
private String type;

@Column(name = "config_data")
@Basic(fetch = FetchType.LAZY)
@Lob
private String configData;

@Column(name = "config_attributes")
@Basic(fetch = FetchType.LAZY)
@Lob
private String configAttributes;

@ManyToOne
@JoinColumn(name = "mpack_ref_id", referencedColumnName = "id", nullable = false)
private BlueprintMpackReferenceEntity mpackReference;

public Long getMpackRefId() {
return mpackRefId;
}

public void setMpackRefId(Long mpackRefId) {
this.mpackRefId = mpackRefId;
}

@Override
public String getType() {
return type;
}

@Override
public void setType(String typeName) {
this.type = typeName;
}

public String getConfigData() {
return configData;
}

@Override
public void setBlueprintName(String blueprintName) {
throw new UnsupportedOperationException();
}

@Override
public String getBlueprintName() {
return getMpackReference().getBlueprint().getBlueprintName();
}

public void setConfigData(String configData) {
this.configData = configData;
}

public String getConfigAttributes() {
return configAttributes;
}

public void setConfigAttributes(String configAttributes) {
this.configAttributes = configAttributes;
}

public BlueprintMpackReferenceEntity getMpackReference() {
return mpackReference;
}

public void setMpackReference(BlueprintMpackReferenceEntity mpackReference) {
this.mpackReference = mpackReference;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.ambari.server.orm.entities;

import java.util.Objects;

import javax.persistence.Column;
import javax.persistence.Id;

public class BlueprintMpackConfigEntityPk {

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This class needs documentation.

@Id
@Column(name = "mpack_ref_id", nullable = false, insertable = true, updatable = false)
private Long mpackRefId;

@Id
@Column(name = "type_name", nullable = false, insertable = true, updatable = false, length = 100)
private String type;

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
BlueprintMpackConfigEntityPk that = (BlueprintMpackConfigEntityPk) o;
return Objects.equals(mpackRefId, that.mpackRefId) &&
Objects.equals(type, that.type);
}

@Override
public int hashCode() {
return Objects.hash(mpackRefId, type);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.ambari.server.orm.entities;

import java.util.ArrayList;
import java.util.Collection;

import javax.persistence.CascadeType;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import javax.persistence.TableGenerator;

@Entity
@Table(name = "blueprint_mpack_reference")
@TableGenerator(name = "blueprint_mpack_reference_id_generator", table = "ambari_sequences", pkColumnName = "sequence_name",
valueColumnName = "sequence_value", pkColumnValue = "blueprint_mpack_ref_id_seq", initialValue = 1)
public class BlueprintMpackReferenceEntity {

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This class needs documentation.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

@Id
@GeneratedValue(strategy = GenerationType.TABLE, generator = "blueprint_mpack_reference_id_generator")
@Column(name = "id", nullable = false, updatable = false)
private Long id;

@Column(name = "mpack_name")

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we have duplicated fields here? Shouldn't this just be a foreign key to an mpack?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wanted to decouple the blueprint lifecycle from the mpack lifecycle. This is a valid case:

  • user posts a blueprint (mpack does not exist at this time as automatic mpack download will happen at cluster installation time)
  • user installs the necessary mpacks (or they are auto-dowloaded at cluster installation)

I am not sure this can be modelled with foreign keys as after the first step there are no mpacks in the DB the FK's can refer to.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is that really a valid use case? Ambari can't auto download the correct management pack because there could be multiple places it's hosted. The mpacks have to be installed first for any blueprint installation, right?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rnettleton @jayush Could you pls. clarify if blueprints have an independent lifecycle from mpacks, such as the user can send in a blueprint referencing an mpack that is not yet installed.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@benyoka , Blueprints should indeed have a separate lifecycle from mpack installs.

A POST of a Blueprint to the Ambari REST API should only cause the document to the validated and persisted. Since the Blueprint can include references to an MPack, these should be validated as much as possible, but no download/install of an MPack should occur due to just a POST of a Blueprint. The MPack installation process, when handled by Blueprints, should only occur when a Cluster Creation Template (referring to a Blueprint) is POST-ed to the Ambari Server.

@jonathan-hurley , We're trying to support multiple deployment models in Blueprints:

  1. Ambari user manually installs a set of mpacks, and then deploys a Blueprint that references these.
  2. Ambari user lists the necessary MPacks in a Blueprint (including download URLs), and expects Ambari to handle the downloading/installation of the mpacks prior to the cluster creation start.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So in the last case, where the mpacks are not already installed, you are saying that you will only pass some basic info about an mpack (which can't be trusted since the only source of truth would be the metadata inside of the downloaded mpack bundle) along with a URL (the only source of truth). In this scenario, nothing is downloaded yet, right?

So, the FK can be empty, but once the mpack is downloaded, should it not have an association made? At the very least, we shouldn't put names in here since they could be wrong compared to what's actually in the mpack.

@benyoka benyoka Jan 19, 2018

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the last case, when a blueprint is saved the only things it knows about an mpack is its name and version (and url in addition). I need to store this information as this will be the basis for linking the blueprint to an actual mpack.

I also need to store information that is not directly related to the mpack but to the blueprint's use of that mpack, e.g. special configuration that applies to services installed from that mpack. This is the other purpose for this table.

Would you agree with the following changes?

  • Adding a nullable FK column that references the mpack table (still need to store mpack_name and mpack_version as those will be the basis of finding the right mpack)
  • renaming the entity to MpackInstanceEntity as MpackReferenceEntity can be misleading.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I'd be good with those changes - and to also document that the mpack name/version is what was supplied and may not be correct (or even change their field names to something that reflects that fact)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1 for adding a nullable mpack_id

private String mpackName;

@Column(name = "mpack_version")
private String mpackVersion;

@Column(name = "mpack_uri")
private String mpackUri;

@OneToMany(cascade = CascadeType.ALL, mappedBy = "mpackReference")
private Collection<BlueprintServiceEntity> serviceInstances = new ArrayList<>();

@OneToMany(cascade = CascadeType.ALL, mappedBy = "mpackReference")
private Collection<BlueprintMpackConfigEntity> configurations = new ArrayList<>();


@ManyToOne
@JoinColumn(name = "blueprint_name", referencedColumnName = "blueprint_name", nullable = false)
private BlueprintEntity blueprint;

public Collection<BlueprintServiceEntity> getServiceInstances() {
return serviceInstances;
}

public void setServiceInstances(Collection<BlueprintServiceEntity> serviceInstances) {
this.serviceInstances = serviceInstances;
}

public String getMpackName() {
return mpackName;
}

public void setMpackName(String mpackName) {
this.mpackName = mpackName;
}

public String getMpackVersion() {
return mpackVersion;
}

public void setMpackVersion(String mpackVersion) {
this.mpackVersion = mpackVersion;
}

public String getMpackUri() {
return mpackUri;
}

public void setMpackUri(String mpackUri) {
this.mpackUri = mpackUri;
}

public Long getId() {
return id;
}

public void setId(Long id) {
this.id = id;
}

public BlueprintEntity getBlueprint() {
return blueprint;
}

public void setBlueprint(BlueprintEntity blueprint) {
this.blueprint = blueprint;
}

public Collection<BlueprintMpackConfigEntity> getConfigurations() {
return configurations;
}

public void setConfigurations(Collection<BlueprintMpackConfigEntity> configurations) {
this.configurations = configurations;
}
}
Loading