diff --git a/.gitignore b/.gitignore index 551b1b5361ce..e09c2eb819c0 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ .classpath .project .settings +*.factorypath target build dependency-reduced-pom.xml @@ -61,5 +62,6 @@ output.xml report.html hadoop-hdds/docs/public +hadoop-ozone/recon/node_modules .mvn diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java index 1b6a2141e0f0..96f19a6b87a8 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java @@ -49,6 +49,8 @@ public class DatanodeDetails extends NodeImpl implements private String hostName; private List ports; private String certSerialId; + private String version; + private long setupTime; /** * Constructs DatanodeDetails instance. DatanodeDetails.Builder is used @@ -59,15 +61,21 @@ public class DatanodeDetails extends NodeImpl implements * @param networkLocation DataNode's network location path * @param ports Ports used by the DataNode * @param certSerialId serial id from SCM issued certificate. + * @param version DataNode's version + * @param setupTime the setup time of DataNode */ + @SuppressWarnings("parameternumber") private DatanodeDetails(UUID uuid, String ipAddress, String hostName, - String networkLocation, List ports, String certSerialId) { + String networkLocation, List ports, String certSerialId, + String version, long setupTime) { super(hostName, networkLocation, NetConstants.NODE_COST_DEFAULT); this.uuid = uuid; this.ipAddress = ipAddress; this.hostName = hostName; this.ports = ports; this.certSerialId = certSerialId; + this.version = version; + this.setupTime = setupTime; } public DatanodeDetails(DatanodeDetails datanodeDetails) { @@ -79,6 +87,8 @@ public DatanodeDetails(DatanodeDetails datanodeDetails) { this.ports = datanodeDetails.ports; this.setNetworkName(datanodeDetails.getNetworkName()); this.setParent(datanodeDetails.getParent()); + this.version = datanodeDetails.version; + this.setupTime = datanodeDetails.setupTime; } /** @@ -207,6 +217,12 @@ public static DatanodeDetails getFromProtoBuf( if (datanodeDetailsProto.hasNetworkLocation()) { builder.setNetworkLocation(datanodeDetailsProto.getNetworkLocation()); } + if (datanodeDetailsProto.hasVersion()) { + builder.setVersion(datanodeDetailsProto.getVersion()); + } + if (datanodeDetailsProto.hasSetupTime()) { + builder.setSetupTime(datanodeDetailsProto.getSetupTime()); + } return builder.build(); } @@ -248,6 +264,13 @@ public HddsProtos.DatanodeDetailsProto getProtoBufMessage() { .setValue(port.getValue()) .build()); } + + if (!Strings.isNullOrEmpty(getVersion())) { + builder.setVersion(getVersion()); + } + + builder.setSetupTime(getSetupTime()); + return builder.build(); } @@ -300,6 +323,8 @@ public static final class Builder { private String networkLocation; private List ports; private String certSerialId; + private String version; + private long setupTime; /** * Default private constructor. To create Builder instance use @@ -388,6 +413,30 @@ public Builder setCertSerialId(String certId) { return this; } + /** + * Sets the DataNode version. + * + * @param ver the version of DataNode. + * + * @return DatanodeDetails.Builder + */ + public Builder setVersion(String ver) { + this.version = ver; + return this; + } + + /** + * Sets the DataNode setup time. + * + * @param time the setup time of DataNode. + * + * @return DatanodeDetails.Builder + */ + public Builder setSetupTime(long time) { + this.setupTime = time; + return this; + } + /** * Builds and returns DatanodeDetails instance. * @@ -399,7 +448,7 @@ public DatanodeDetails build() { networkLocation = NetConstants.DEFAULT_RACK; } DatanodeDetails dn = new DatanodeDetails(id, ipAddress, hostName, - networkLocation, ports, certSerialId); + networkLocation, ports, certSerialId, version, setupTime); if (networkName != null) { dn.setNetworkName(networkName); } @@ -505,4 +554,40 @@ public String getCertSerialId() { public void setCertSerialId(String certSerialId) { this.certSerialId = certSerialId; } + + /** + * Returns the DataNode version. + * + * @return DataNode version + */ + public String getVersion() { + return version; + } + + /** + * Set DataNode version. + * + * @param version DataNode version + */ + public void setVersion(String version) { + this.version = version; + } + + /** + * Returns the DataNode setup time. + * + * @return DataNode setup time + */ + public long getSetupTime() { + return setupTime; + } + + /** + * Set DataNode setup time. + * + * @param setupTime DataNode setup time + */ + public void setSetupTime(long setupTime) { + this.setupTime = setupTime; + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index 7e896e715598..08eef6f88eac 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -65,6 +65,8 @@ import static org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest.getEncodedString; import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_DATANODE_PLUGINS_KEY; import static org.apache.hadoop.util.ExitUtil.terminate; + +import org.apache.hadoop.util.Time; import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -191,6 +193,9 @@ public void start() { datanodeDetails = initializeDatanodeDetails(); datanodeDetails.setHostName(hostname); datanodeDetails.setIpAddress(ip); + datanodeDetails.setVersion( + HddsVersionInfo.HDDS_VERSION_INFO.getVersion()); + datanodeDetails.setSetupTime(Time.now()); TracingUtil.initTracing( "HddsDatanodeService." + datanodeDetails.getUuidString() .substring(0, 8), conf); diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto index 23cc9cbf6ced..243e8ecaced7 100644 --- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto +++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto @@ -43,6 +43,8 @@ message DatanodeDetailsProto { // network name, can be Ip address or host name, depends optional string networkName = 6; optional string networkLocation = 7; // Network topology location + optional string version = 8; // Datanode version + optional int64 setupTime = 9; // TODO(runzhiwang): when uuid is gone, specify 1 as the index of uuid128 and mark as required optional UUID uuid128 = 100; // UUID with 128 bits assigned to the Datanode. } diff --git a/hadoop-hdds/interface-client/src/main/proto/proto.lock b/hadoop-hdds/interface-client/src/main/proto/proto.lock index afdaf9618260..b27896c655e3 100644 --- a/hadoop-hdds/interface-client/src/main/proto/proto.lock +++ b/hadoop-hdds/interface-client/src/main/proto/proto.lock @@ -1530,6 +1530,16 @@ "name": "networkLocation", "type": "string" }, + { + "id": 8, + "name": "version", + "type": "string" + }, + { + "id": 9, + "name": "setupTime", + "type": "int64" + }, { "id": 100, "name": "uuid128", @@ -1925,4 +1935,4 @@ } } ] -} \ No newline at end of file +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NodeEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NodeEndpoint.java index 2c017491f59d..42832debfee1 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NodeEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/NodeEndpoint.java @@ -121,6 +121,8 @@ public Response getDatanodes() { .withPipelines(pipelines) .withLeaderCount(leaderCount.get()) .withUUid(datanode.getUuidString()) + .withVersion(datanode.getVersion()) + .withSetupTime(datanode.getSetupTime()) .build()); }); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodeMetadata.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodeMetadata.java index 02d9ae811829..542654e96e2e 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodeMetadata.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/DatanodeMetadata.java @@ -55,6 +55,12 @@ public final class DatanodeMetadata { @XmlElement(name = "leaderCount") private int leaderCount; + @XmlElement(name = "version") + private String version; + + @XmlElement(name = "setupTime") + private long setupTime; + private DatanodeMetadata(Builder builder) { this.hostname = builder.hostname; this.uuid = builder.uuid; @@ -64,6 +70,8 @@ private DatanodeMetadata(Builder builder) { this.pipelines = builder.pipelines; this.containers = builder.containers; this.leaderCount = builder.leaderCount; + this.version = builder.version; + this.setupTime = builder.setupTime; } public String getHostname() { @@ -98,6 +106,14 @@ public String getUuid() { return uuid; } + public String getVersion() { + return version; + } + + public long getSetupTime() { + return setupTime; + } + /** * Returns new builder class that builds a DatanodeMetadata. * @@ -120,6 +136,8 @@ public static final class Builder { private List pipelines; private int containers; private int leaderCount; + private String version; + private long setupTime; public Builder() { this.containers = 0; @@ -167,6 +185,16 @@ public Builder withUUid(String uuid) { return this; } + public Builder withVersion(String version) { + this.version = version; + return this; + } + + public Builder withSetupTime(long setupTime) { + this.setupTime = setupTime; + return this; + } + /** * Constructs DatanodeMetadata. * diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json index 82fae3735b12..d8d6eac55a9f 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json @@ -41,7 +41,9 @@ } ], "containers": 80, - "leaderCount": 2 + "leaderCount": 2, + "version": "0.6.0-SNAPSHOT", + "setupTime": 1574728775759 }, { "hostname": "localhost2.storage.enterprise.com", @@ -68,7 +70,9 @@ } ], "containers": 8192, - "leaderCount": 1 + "leaderCount": 1, + "version": "0.6.0-SNAPSHOT", + "setupTime": 1574724805059 }, { "hostname": "localhost3.storage.enterprise.com", @@ -101,7 +105,9 @@ } ], "containers": 43, - "leaderCount": 2 + "leaderCount": 2, + "version": "0.6.0-SNAPSHOT", + "setupTime": 1343544679543 }, { "hostname": "localhost4.storage.enterprise.com", @@ -115,7 +121,9 @@ }, "pipelines": [], "containers": 0, - "leaderCount": 0 + "leaderCount": 0, + "version": "0.6.0-SNAPSHOT", + "setupTime": 1074724802059 }, { "hostname": "localhost5.storage.enterprise.com", @@ -142,7 +150,9 @@ } ], "containers": 643, - "leaderCount": 2 + "leaderCount": 2, + "version": "0.6.0-SNAPSHOT", + "setupTime": 1574724816029 }, { "hostname": "localhost6.storage.enterprise.com", @@ -169,7 +179,9 @@ } ], "containers": 5, - "leaderCount": 1 + "leaderCount": 1, + "version": "0.6.0-SNAPSHOT", + "setupTime": 1574724802059 }, { "hostname": "localhost7.storage.enterprise.com", @@ -202,7 +214,9 @@ } ], "containers": 64, - "leaderCount": 2 + "leaderCount": 2, + "version": "0.6.0-SNAPSHOT", + "setupTime": 1574724676009 }, { "hostname": "localhost8.storage.enterprise.com", @@ -229,7 +243,9 @@ } ], "containers": 21, - "leaderCount": 1 + "leaderCount": 1, + "version": "0.6.0-SNAPSHOT", + "setupTime": 1574724276050 }, { "hostname": "localhost9.storage.enterprise.com", @@ -256,7 +272,9 @@ } ], "containers": 897, - "leaderCount": 1 + "leaderCount": 1, + "version": "0.6.0-SNAPSHOT", + "setupTime": 1574724573011 }, { "hostname": "localhost10.storage.enterprise.com", @@ -289,7 +307,9 @@ } ], "containers": 6754, - "leaderCount": 2 + "leaderCount": 2, + "version": "0.6.0-SNAPSHOT", + "setupTime": 1574723756059 }, { "hostname": "localhost11.storage.enterprise.com", @@ -316,7 +336,9 @@ } ], "containers": 78, - "leaderCount": 2 + "leaderCount": 2, + "version": "0.6.0-SNAPSHOT", + "setupTime": 1474724705783 }, { "hostname": "localhost12.storage.enterprise.com", @@ -343,7 +365,9 @@ } ], "containers": 543, - "leaderCount": 1 + "leaderCount": 1, + "version": "0.6.0-SNAPSHOT", + "setupTime": 1574724706232 } ] }, diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/multiSelect/multiSelect.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/multiSelect/multiSelect.tsx index 19005dddd11e..417c2efdcb7a 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/multiSelect/multiSelect.tsx +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/multiSelect/multiSelect.tsx @@ -36,6 +36,7 @@ interface IMultiSelectProps extends ReactSelectProps { options: IOption[]; allowSelectAll: boolean; allOption?: IOption; + maxShowValues?: number; } const defaultProps = { @@ -48,7 +49,7 @@ const defaultProps = { export class MultiSelect extends PureComponent { static defaultProps = defaultProps; render() { - const {allowSelectAll, allOption, options, onChange} = this.props; + const {allowSelectAll, allOption, options, maxShowValues = 5, onChange} = this.props; if (allowSelectAll) { const Option = (props: OptionProps) => { return ( @@ -70,7 +71,7 @@ export class MultiSelect extends PureComponent { let toBeRendered = children; if (currentValues.some(val => val.value === allOption!.value) && children) { toBeRendered = allOption!.label; - } else if (currentValues.length >= 5) { + } else if (currentValues.length > maxShowValues) { toBeRendered = `${currentValues.length} selected`; } diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/datanodes.less b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/datanodes.less index 4a3cdf5accc9..10ec907a7334 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/datanodes.less +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/datanodes.less @@ -22,4 +22,18 @@ margin-bottom: 5px; } } + + .filter-block { + font-size: 14px; + font-weight: normal; + display: inline-block; + margin-left: 20px; + } + + .multi-select-container { + padding-left: 5px; + margin-right: 5px; + display: inline-block; + min-width: 200px; + } } diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/datanodes.tsx b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/datanodes.tsx index bfba82a5bebb..877ebf9f2c75 100644 --- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/datanodes.tsx +++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/datanodes.tsx @@ -27,6 +27,8 @@ import {DatanodeStatus, IStorageReport} from 'types/datanode.types'; import './datanodes.less'; import {AutoReloadHelper} from 'utils/autoReloadHelper'; import AutoReloadPanel from 'components/autoReloadPanel/autoReloadPanel'; +import {MultiSelect, IOption} from 'components/multiSelect/multiSelect'; +import {ActionMeta, ValueType} from 'react-select'; import {showDataFetchError} from 'utils/common'; interface IDatanodeResponse { @@ -38,6 +40,8 @@ interface IDatanodeResponse { containers: number; leaderCount: number; uuid: string; + version: string; + setupTime: number; } interface IDatanodesResponse { @@ -56,6 +60,8 @@ interface IDatanode { containers: number; leaderCount: number; uuid: string; + version: string; + setupTime: number; } interface IPipeline { @@ -70,6 +76,8 @@ interface IDatanodesState { dataSource: IDatanode[]; totalCount: number; lastUpdated: number; + selectedColumns: IOption[]; + columnOptions: IOption[]; } const renderDatanodeStatus = (status: DatanodeStatus) => { @@ -89,6 +97,7 @@ const COLUMNS = [ title: 'Status', dataIndex: 'state', key: 'state', + isVisible: true, render: (text: DatanodeStatus) => renderDatanodeStatus(text), sorter: (a: IDatanode, b: IDatanode) => a.state.localeCompare(b.state) }, @@ -96,6 +105,7 @@ const COLUMNS = [ title: 'Uuid', dataIndex: 'uuid', key: 'uuid', + isVisible: true, sorter: (a: IDatanode, b: IDatanode) => a.uuid.localeCompare(b.uuid), defaultSortOrder: 'ascend' as const }, @@ -103,6 +113,7 @@ const COLUMNS = [ title: 'Hostname', dataIndex: 'hostname', key: 'hostname', + isVisible: true, sorter: (a: IDatanode, b: IDatanode) => a.hostname.localeCompare(b.hostname), defaultSortOrder: 'ascend' as const }, @@ -110,6 +121,7 @@ const COLUMNS = [ title: 'Storage Capacity', dataIndex: 'storageUsed', key: 'storageUsed', + isVisible: true, sorter: (a: IDatanode, b: IDatanode) => a.storageRemaining - b.storageRemaining, render: (text: string, record: IDatanode) => ( a.lastHeartbeat - b.lastHeartbeat, render: (heartbeat: number) => { return heartbeat > 0 ? moment(heartbeat).format('lll') : 'NA'; @@ -129,6 +142,7 @@ const COLUMNS = [ title: 'Pipeline ID(s)', dataIndex: 'pipelines', key: 'pipelines', + isVisible: true, render: (pipelines: IPipeline[], record: IDatanode) => { return (
@@ -158,16 +172,46 @@ const COLUMNS = [ , dataIndex: 'leaderCount', key: 'leaderCount', + isVisible: true, sorter: (a: IDatanode, b: IDatanode) => a.leaderCount - b.leaderCount }, { title: 'Containers', dataIndex: 'containers', key: 'containers', + isVisible: true, sorter: (a: IDatanode, b: IDatanode) => a.containers - b.containers + }, + { + title: 'Version', + dataIndex: 'version', + key: 'version', + isVisible: false, + sorter: (a: IDatanode, b: IDatanode) => a.version.localeCompare(b.version), + defaultSortOrder: 'ascend' as const + }, + { + title: 'SetupTime', + dataIndex: 'setupTime', + key: 'setupTime', + isVisible: false, + sorter: (a: IDatanode, b: IDatanode) => a.setupTime - b.setupTime, + render: (uptime: number) => { + return uptime > 0 ? moment(uptime).format('lll') : 'NA'; + } } ]; +const allColumnsOption: IOption = { + label: 'Select all', + value: '*' +}; + +const defaultColumns: IOption[] = COLUMNS.map(column => ({ + label: column.key, + value: column.key +})); + export class Datanodes extends React.Component, IDatanodesState> { autoReload: AutoReloadHelper; @@ -177,11 +221,20 @@ export class Datanodes extends React.Component, IDatanode loading: false, dataSource: [], totalCount: 0, - lastUpdated: 0 + lastUpdated: 0, + selectedColumns: [], + columnOptions: defaultColumns }; this.autoReload = new AutoReloadHelper(this._loadData); } + _handleColumnChange = (selected: ValueType, _action: ActionMeta) => { + const selectedColumns = (selected as IOption[]); + this.setState({ + selectedColumns + }); + }; + _loadData = () => { this.setState({ loading: true @@ -201,14 +254,23 @@ export class Datanodes extends React.Component, IDatanode storageRemaining: datanode.storageReport.remaining, pipelines: datanode.pipelines, containers: datanode.containers, - leaderCount: datanode.leaderCount + leaderCount: datanode.leaderCount, + version: datanode.version, + setupTime: datanode.setupTime }; }); + const selectedColumns: IOption[] = COLUMNS.filter(column => column.isVisible).map(column => ({ + label: column.key, + value: column.key + })); + this.setState({ loading: false, dataSource, totalCount, lastUpdated: Number(moment()) + }, () => { + this._handleColumnChange(selectedColumns, {action: 'select-option'}); }); }).catch(error => { this.setState({ @@ -233,7 +295,7 @@ export class Datanodes extends React.Component, IDatanode }; render() { - const {dataSource, loading, totalCount, lastUpdated} = this.state; + const {dataSource, loading, totalCount, lastUpdated, selectedColumns, columnOptions} = this.state; const paginationConfig: PaginationConfig = { showTotal: (total: number, range) => `${range[0]}-${range[1]} of ${total} datanodes`, showSizeChanger: true, @@ -243,10 +305,38 @@ export class Datanodes extends React.Component, IDatanode
Datanodes ({totalCount}) - +
+ Columns +
+
+
- +
+ selectedColumns.some(e => e.value === column.key) + )} + loading={loading} + pagination={paginationConfig} + rowKey='hostname' + /> );