Skip to content

HDDS-9807. Consider volume committed space when checking if datanode can host new container #5721

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adoroszlai merged 10 commits into from
Dec 20, 2023
Merged

HDDS-9807. Consider volume committed space when checking if datanode can host new container #5721

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
72d0d69
HDDS-9807. use the same algorithm for SCM to detect an hdds volume av…
Nov 30, 2023
cfcc024
HDDS-9807. update remaining volume size (- committedBytes) instead of…
Dec 6, 2023
db1305c
Revert "HDDS-9807. update remaining volume size (- committedBytes) in…
Dec 7, 2023
9d8c56d
HDDS-9807. add committed storage bytes count to the recon datanode re…
Dec 8, 2023
5fe6ceb
HDDS-9807. add committed bytes count to volume info metrics
Dec 8, 2023
59ece8d
HDDS-9807. provide datanode freeSpaceToSpare through heartbeat reques…
Dec 10, 2023
58b6aba
HDDS-9807. fix test to check datanode availability (configuration is …
Dec 10, 2023
873aeaf
HDDS-9807. add FreeSpaceToSpare datanode volume metric to the datanod…
Dec 11, 2023
a6ff69a
HDDS-9807. align datanode usageinfo sub-command output
Dec 11, 2023
529c0cb
Merge remote-tracking branch 'origin/master' into HDDS-9807-scm-corre…
adoroszlai Dec 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 53 additions & 3 deletions ...ce/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.container.common.impl;

import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.proto.
StorageContainerDatanodeProtocolProtos.MetadataStorageReportProto;
import org.apache.hadoop.hdds.protocol.proto.
Expand All @@ -27,6 +28,7 @@
StorageContainerDatanodeProtocolProtos.StorageTypeProto;
import org.apache.hadoop.ozone.container.common.interfaces
.StorageLocationReportMXBean;
import org.apache.hadoop.ozone.container.common.volume.VolumeUsage;

import java.io.IOException;

Expand All @@ -42,17 +44,22 @@ public final class StorageLocationReport implements
private final long capacity;
private final long scmUsed;
private final long remaining;
private final long committed;
private final long freeSpaceToSpare;
private final StorageType storageType;
private final String storageLocation;

@SuppressWarnings("checkstyle:parameternumber")
private StorageLocationReport(String id, boolean failed, long capacity,
long scmUsed, long remaining, StorageType storageType,
String storageLocation) {
long scmUsed, long remaining, long committed, long freeSpaceToSpare,
StorageType storageType, String storageLocation) {
this.id = id;
this.failed = failed;
this.capacity = capacity;
this.scmUsed = scmUsed;
this.remaining = remaining;
this.committed = committed;
this.freeSpaceToSpare = freeSpaceToSpare;
this.storageType = storageType;
this.storageLocation = storageLocation;
}
Expand Down Expand Up @@ -82,6 +89,16 @@ public long getRemaining() {
return remaining;
}

@Override
public long getCommitted() {
return committed;
}

@Override
public long getFreeSpaceToSpare() {
return freeSpaceToSpare;
}

@Override
public String getStorageLocation() {
return storageLocation;
Expand Down Expand Up @@ -157,14 +174,22 @@ private static StorageType getStorageType(StorageTypeProto proto) throws
* @throws IOException In case, the storage type specified is invalid.
*/
public StorageReportProto getProtoBufMessage() throws IOException {
return getProtoBufMessage(null);
}

public StorageReportProto getProtoBufMessage(ConfigurationSource conf)
throws IOException {
StorageReportProto.Builder srb = StorageReportProto.newBuilder();
return srb.setStorageUuid(getId())
.setCapacity(getCapacity())
.setScmUsed(getScmUsed())
.setRemaining(getRemaining())
.setCommitted(getCommitted())
.setStorageType(getStorageTypeProto())
.setStorageLocation(getStorageLocation())
.setFailed(isFailed())
.setFreeSpaceToSpare(conf != null ?
VolumeUsage.getMinVolumeFreeSpace(conf, getCapacity()) : 0)
.build();
}

Expand Down Expand Up @@ -266,6 +291,8 @@ public static class Builder {
private long capacity;
private long scmUsed;
private long remaining;
private long committed;
private long freeSpaceToSpare;
private StorageType storageType;
private String storageLocation;

Expand Down Expand Up @@ -334,6 +361,29 @@ public Builder setStorageType(StorageType storageTypeValue) {
return this;
}

/**
* Sets the committed bytes count.
* (bytes for previously created containers)
* @param committed previously created containers size
* @return StorageLocationReport.Builder
*/
public Builder setCommitted(long committed) {
this.committed = committed;
return this;
}

/**
* Sets the free space available to spare.
* (depends on datanode volume config,
* consider 'hdds.datanode.volume.min.*' configuration properties)
* @param freeSpaceToSpare the size of free volume space available to spare
* @return StorageLocationReport.Builder
*/
public Builder setFreeSpaceToSpare(long freeSpaceToSpare) {
this.freeSpaceToSpare = freeSpaceToSpare;
return this;
}

/**
* Sets the storageLocation.
*
Expand All @@ -352,7 +402,7 @@ public Builder setStorageLocation(String storageLocationValue) {
*/
public StorageLocationReport build() {
return new StorageLocationReport(id, failed, capacity, scmUsed,
remaining, storageType, storageLocation);
remaining, committed, freeSpaceToSpare, storageType, storageLocation);
}

}
Expand Down
4 changes: 4 additions & 0 deletions ...java/org/apache/hadoop/ozone/container/common/interfaces/StorageLocationReportMXBean.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ public interface StorageLocationReportMXBean {

long getRemaining();
Copy link
Contributor

@xichen01 xichen01 Dec 5, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@vtutrinov @adoroszlai
What is the definition of remaining, I noticed that remaining is actually Volume available. available=capacity - reserved - used.

  • So does committed have to be reported to the SCM separately.
  • Or is it added to remaining, which is defined as the amount of space on the Volume that is guaranteed to be available for data storage? available=capacity - reserved - used - commited.
    We use available in a lot of places, and it seems like we should take commited into available.

ozone/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java

Line 473 in 9b3d045

remaining = volumeInfo.get().getAvailable();

Copy link
Contributor Author

@vtutrinov vtutrinov Dec 5, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

seems it makes sense, I will try to check the approach

Copy link
Contributor Author

@vtutrinov vtutrinov Dec 6, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@xichen01
I've pushed a new commit with the proposed approach and the build is green, as such as the manual test from the jira ticket. But I'm concerned about the datanode storage metrics (output of the ozone admin datanode usageinfo command) and recon's reports - from the user's perspective it will seen contr ordinal:

  • I've written a 200KiB key to the EC-bucket with rs-6-3-1024k replication (the cluster has 10 datanodes with 2Gb storages mounted to /data)
  • datanodes report before the key was written:
Usage Information (1 Datanodes)

UUID         : 8ac06e73-a434-4fd3-9be5-bc271b8f6bf7 
IP Address   : 172.23.0.15 
Hostname     : ozone-datanode10-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 31571968 B (30.11 MB) 
Total Used % : 1.47% 
Ozone Used   : 0 B (0 B) 
Ozone Used % : 0.00% 
Remaining    : 2115911680 B (1.97 GB) 
Remaining %  : 98.53% 
Container(s) : 0 

Usage Information (1 Datanodes)

UUID         : 9e3bf76a-8c98-4fa3-b034-c68973d7560b 
IP Address   : 172.23.0.10 
Hostname     : ozone-datanode3-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 31571968 B (30.11 MB) 
Total Used % : 1.47% 
Ozone Used   : 0 B (0 B) 
Ozone Used % : 0.00% 
Remaining    : 2115911680 B (1.97 GB) 
Remaining %  : 98.53% 
Container(s) : 0 

Usage Information (1 Datanodes)

UUID         : a5df4b7d-92b3-407a-b56d-aaeb924f0cd6 
IP Address   : 172.23.0.7 
Hostname     : ozone-datanode2-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 27365376 B (26.10 MB) 
Total Used % : 1.27% 
Ozone Used   : 0 B (0 B) 
Ozone Used % : 0.00% 
Remaining    : 2120118272 B (1.97 GB) 
Remaining %  : 98.73% 
Container(s) : 0 

Usage Information (1 Datanodes)

UUID         : e5a630fc-ce77-4211-89dc-15ab0ca67fde 
IP Address   : 172.23.0.6 
Hostname     : ozone-datanode8-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 31571968 B (30.11 MB) 
Total Used % : 1.47% 
Ozone Used   : 0 B (0 B) 
Ozone Used % : 0.00% 
Remaining    : 2115911680 B (1.97 GB) 
Remaining %  : 98.53% 
Container(s) : 0 

Usage Information (1 Datanodes)

UUID         : 2d00f163-7f22-4395-b82b-e8353c74698b 
IP Address   : 172.23.0.5 
Hostname     : ozone-datanode7-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 31571968 B (30.11 MB) 
Total Used % : 1.47% 
Ozone Used   : 0 B (0 B) 
Ozone Used % : 0.00% 
Remaining    : 2115911680 B (1.97 GB) 
Remaining %  : 98.53% 
Container(s) : 0 

Usage Information (1 Datanodes)

UUID         : 3a22dc09-acc5-4270-8007-135056031ed9 
IP Address   : 172.23.0.12 
Hostname     : ozone-datanode9-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 31571968 B (30.11 MB) 
Total Used % : 1.47% 
Ozone Used   : 0 B (0 B) 
Ozone Used % : 0.00% 
Remaining    : 2115911680 B (1.97 GB) 
Remaining %  : 98.53% 
Container(s) : 0 

Usage Information (1 Datanodes)

UUID         : 4478a423-0416-429e-b41c-5e58b8ec7ed5 
IP Address   : 172.23.0.16 
Hostname     : ozone-datanode5-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 31571968 B (30.11 MB) 
Total Used % : 1.47% 
Ozone Used   : 0 B (0 B) 
Ozone Used % : 0.00% 
Remaining    : 2115911680 B (1.97 GB) 
Remaining %  : 98.53% 
Container(s) : 0 

Usage Information (1 Datanodes)

UUID         : 5ffda4af-e5dd-4b11-a945-7fef0b53356b 
IP Address   : 172.23.0.17 
Hostname     : ozone-datanode1-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 31571968 B (30.11 MB) 
Total Used % : 1.47% 
Ozone Used   : 0 B (0 B) 
Ozone Used % : 0.00% 
Remaining    : 2115911680 B (1.97 GB) 
Remaining %  : 98.53% 
Container(s) : 0 

Usage Information (1 Datanodes)

UUID         : 6fe5b830-5031-4a4c-a8fa-ae909c55d187 
IP Address   : 172.23.0.3 
Hostname     : ozone-datanode6-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 31571968 B (30.11 MB) 
Total Used % : 1.47% 
Ozone Used   : 0 B (0 B) 
Ozone Used % : 0.00% 
Remaining    : 2115911680 B (1.97 GB) 
Remaining %  : 98.53% 
Container(s) : 0 

Usage Information (1 Datanodes)

UUID         : 7ce23e7c-41bf-4c5e-aea7-63af06049ad0 
IP Address   : 172.23.0.19 
Hostname     : ozone-datanode4-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 31571968 B (30.11 MB) 
Total Used % : 1.47% 
Ozone Used   : 0 B (0 B) 
Ozone Used % : 0.00% 
Remaining    : 2115911680 B (1.97 GB) 
Remaining %  : 98.53% 
Container(s) : 0
  • datanode report after the key was written:
Usage Information (1 Datanodes)

UUID         : 8ac06e73-a434-4fd3-9be5-bc271b8f6bf7 
IP Address   : 172.23.0.15 
Hostname     : ozone-datanode10-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 1252982784 B (1.17 GB) 
Total Used % : 58.35% 
Ozone Used   : 0 B (0 B) 
Ozone Used % : 0.00% 
Remaining    : 894500864 B (853.06 MB) 
Remaining %  : 41.65% 
Container(s) : 1 

Usage Information (1 Datanodes)

UUID         : 9e3bf76a-8c98-4fa3-b034-c68973d7560b 
IP Address   : 172.23.0.10 
Hostname     : ozone-datanode3-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 1252982784 B (1.17 GB) 
Total Used % : 58.35% 
Ozone Used   : 0 B (0 B) 
Ozone Used % : 0.00% 
Remaining    : 894500864 B (853.06 MB) 
Remaining %  : 41.65% 
Container(s) : 1 

Usage Information (1 Datanodes)

UUID         : a5df4b7d-92b3-407a-b56d-aaeb924f0cd6 
IP Address   : 172.23.0.7 
Hostname     : ozone-datanode2-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 1248776192 B (1.16 GB) 
Total Used % : 58.15% 
Ozone Used   : 204800 B (200 KB) 
Ozone Used % : 0.01% 
Remaining    : 898707456 B (857.07 MB) 
Remaining %  : 41.85% 
Container(s) : 1 

Usage Information (1 Datanodes)

UUID         : e5a630fc-ce77-4211-89dc-15ab0ca67fde 
IP Address   : 172.23.0.6 
Hostname     : ozone-datanode8-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 31596544 B (30.13 MB) 
Total Used % : 1.47% 
Ozone Used   : 0 B (0 B) 
Ozone Used % : 0.00% 
Remaining    : 2115887104 B (1.97 GB) 
Remaining %  : 98.53% 
Container(s) : 0 

Usage Information (1 Datanodes)

UUID         : 2d00f163-7f22-4395-b82b-e8353c74698b 
IP Address   : 172.23.0.5 
Hostname     : ozone-datanode7-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 1252982784 B (1.17 GB) 
Total Used % : 58.35% 
Ozone Used   : 204800 B (200 KB) 
Ozone Used % : 0.01% 
Remaining    : 894500864 B (853.06 MB) 
Remaining %  : 41.65% 
Container(s) : 1 

Usage Information (1 Datanodes)

UUID         : 3a22dc09-acc5-4270-8007-135056031ed9 
IP Address   : 172.23.0.12 
Hostname     : ozone-datanode9-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 1252982784 B (1.17 GB) 
Total Used % : 58.35% 
Ozone Used   : 0 B (0 B) 
Ozone Used % : 0.00% 
Remaining    : 894500864 B (853.06 MB) 
Remaining %  : 41.65% 
Container(s) : 1 

Usage Information (1 Datanodes)

UUID         : 4478a423-0416-429e-b41c-5e58b8ec7ed5 
IP Address   : 172.23.0.16 
Hostname     : ozone-datanode5-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 1252982784 B (1.17 GB) 
Total Used % : 58.35% 
Ozone Used   : 0 B (0 B) 
Ozone Used % : 0.00% 
Remaining    : 894500864 B (853.06 MB) 
Remaining %  : 41.65% 
Container(s) : 1 

Usage Information (1 Datanodes)

UUID         : 5ffda4af-e5dd-4b11-a945-7fef0b53356b 
IP Address   : 172.23.0.17 
Hostname     : ozone-datanode1-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 1252982784 B (1.17 GB) 
Total Used % : 58.35% 
Ozone Used   : 204800 B (200 KB) 
Ozone Used % : 0.01% 
Remaining    : 894500864 B (853.06 MB) 
Remaining %  : 41.65% 
Container(s) : 1 

Usage Information (1 Datanodes)

UUID         : 6fe5b830-5031-4a4c-a8fa-ae909c55d187 
IP Address   : 172.23.0.3 
Hostname     : ozone-datanode6-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 1252982784 B (1.17 GB) 
Total Used % : 58.35% 
Ozone Used   : 204800 B (200 KB) 
Ozone Used % : 0.01% 
Remaining    : 894500864 B (853.06 MB) 
Remaining %  : 41.65% 
Container(s) : 1 

Usage Information (1 Datanodes)

UUID         : 7ce23e7c-41bf-4c5e-aea7-63af06049ad0 
IP Address   : 172.23.0.19 
Hostname     : ozone-datanode4-1.ozone_default 
Capacity     : 2147483648 B (2 GB) 
Total Used   : 1252982784 B (1.17 GB) 
Total Used % : 58.35% 
Ozone Used   : 0 B (0 B) 
Ozone Used % : 0.00% 
Remaining    : 894500864 B (853.06 MB) 
Remaining %  : 41.65% 
Container(s) : 1

WHAT ??? 200KiB written key has changed the remaining storage size of 9 datanodes from 1.97GiB to 853MB, looks too strange, doesn't it?
I would report the committed bytes as a separate report data field as the remaining storage size will be more obvious for the user and the cluster admins

Copy link
Contributor

@xichen01 xichen01 Dec 6, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this is because we include commited in the remaining, so when a Container is created, the space that the Container might use will be deducted from the remaining.

If we don't include commited in the remaining, the problem is that the datanode list command shows that there is enough space remaining to create a Container, but in reality it is not possible to create a new Container, because inside the Datanode, the creation of a Container has to take into account commited.

Copy link
Contributor Author

@vtutrinov vtutrinov Dec 6, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Exactly, and that's why I would provide the committed bytes count in reports (e.g. in datanode heartbeat requests, datanode usageifo or in datanode list cli commands) as a separate field and don't mix it with the remaining bytes count (Devide et impera policy :) )

Copy link
Contributor

@xichen01 xichen01 Dec 7, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, it looks like your initial solution is the more appropriate choice.


long getCommitted();

long getFreeSpaceToSpare();

String getStorageLocation();

String getStorageTypeName();
Expand Down
8 changes: 4 additions & 4 deletions ...e/src/main/java/org/apache/hadoop/ozone/container/common/volume/AvailableSpaceFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* Filter for selecting volumes with enough space for a new container.
* Keeps track of ineligible volumes for logging/debug purposes.
*/
class AvailableSpaceFilter implements Predicate<HddsVolume> {
public class AvailableSpaceFilter implements Predicate<HddsVolume> {

private final long requiredSpace;
private final Map<HddsVolume, AvailableSpace> fullVolumes =
Expand All @@ -42,10 +42,10 @@ public boolean test(HddsVolume vol) {
long free = vol.getAvailable();
long committed = vol.getCommittedBytes();
long available = free - committed;
long volumeFreeSpace =
long volumeFreeSpaceToSpare =
VolumeUsage.getMinVolumeFreeSpace(vol.getConf(), volumeCapacity);
boolean hasEnoughSpace =
available > Math.max(requiredSpace, volumeFreeSpace);
boolean hasEnoughSpace = VolumeUsage.hasVolumeEnoughSpace(free, committed,
requiredSpace, volumeFreeSpaceToSpare);

mostAvailableSpace = Math.max(available, mostAvailableSpace);

Expand Down
4 changes: 4 additions & 0 deletions ...rvice/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -464,6 +464,7 @@ public StorageLocationReport[] getStorageReport() {
long scmUsed = 0;
long remaining = 0;
long capacity = 0;
long committed = 0;
String rootDir = "";
failed = true;
if (volumeInfo.isPresent()) {
Expand All @@ -472,6 +473,8 @@ public StorageLocationReport[] getStorageReport() {
scmUsed = volumeInfo.get().getScmUsed();
remaining = volumeInfo.get().getAvailable();
capacity = volumeInfo.get().getCapacity();
committed = (volume instanceof HddsVolume) ?
((HddsVolume) volume).getCommittedBytes() : 0;
failed = false;
} catch (UncheckedIOException ex) {
LOG.warn("Failed to get scmUsed and remaining for container " +
Expand All @@ -491,6 +494,7 @@ public StorageLocationReport[] getStorageReport() {
.setCapacity(capacity)
.setRemaining(remaining)
.setScmUsed(scmUsed)
.setCommitted(committed)
.setStorageType(volume.getStorageType());
StorageLocationReport r = builder.build();
reports[counter++] = r;
Expand Down
5 changes: 5 additions & 0 deletions ...vice/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,4 +142,9 @@ public long getTotalCapacity() {
return (getUsed() + getAvailable() + getReserved());
}

@Metric("Returns the Committed bytes of the Volume")
public long getCommitted() {
return volume.getCommittedBytes();
}

}
8 changes: 8 additions & 0 deletions ...er-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,14 @@ public static long getMinVolumeFreeSpace(ConfigurationSource conf,

}

public static boolean hasVolumeEnoughSpace(long volumeAvailableSpace,
long volumeCommittedBytesCount,
long requiredSpace,
long volumeFreeSpaceToSpare) {
return (volumeAvailableSpace - volumeCommittedBytesCount) >
Math.max(requiredSpace, volumeFreeSpaceToSpare);
}

/**
* Class representing precomputed space values of a volume.
* This class is intended to store precomputed values, such as capacity
Expand Down
2 changes: 1 addition & 1 deletion ...ner-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -524,7 +524,7 @@ public StorageContainerDatanodeProtocolProtos.NodeReportProto getNodeReport()
= StorageContainerDatanodeProtocolProtos.
NodeReportProto.newBuilder();
for (int i = 0; i < reports.length; i++) {
nrb.addStorageReport(reports[i].getProtoBufMessage());
nrb.addStorageReport(reports[i].getProtoBufMessage(config));
}

StorageLocationReport[] metaReports = metaVolumeSet.getStorageReport();
Expand Down
6 changes: 6 additions & 0 deletions hadoop-hdds/docs/themes/ozonedoc/static/swagger-resources/recon-api.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1433,6 +1433,9 @@ components:
remaining:
type: number
example: 1080410456064
committed:
type: number
example: 1080410456
containers:
type: integer
example: 26
Expand Down Expand Up @@ -1480,6 +1483,9 @@ components:
remaining:
type: number
example: 270071111680
committed:
type: number
example: 27007111
pipelines:
type: array
items:
Expand Down
2 changes: 2 additions & 0 deletions hadoop-hdds/interface-client/src/main/proto/hdds.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,8 @@ message DatanodeUsageInfoProto {
optional int64 remaining = 3;
optional DatanodeDetailsProto node = 4;
optional int64 containerCount = 5;
optional int64 committed = 6;
optional int64 freeSpaceToSpare = 7;
}

/**
Expand Down
2 changes: 2 additions & 0 deletions hadoop-hdds/interface-server/src/main/proto/ScmServerDatanodeHeartbeatProtocol.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,8 @@ message StorageReportProto {
optional uint64 remaining = 5 [default = 0];
optional StorageTypeProto storageType = 6 [default = DISK];
optional bool failed = 7 [default = false];
optional uint64 committed = 8 [default = 0];
optional uint64 freeSpaceToSpare = 9 [default = 0];
}

message MetadataStorageReportProto {
Expand Down
13 changes: 9 additions & 4 deletions ...op-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.node.NodeStatus;
import org.apache.hadoop.ozone.container.common.volume.VolumeUsage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -274,7 +275,7 @@ public List<DatanodeDetails> filterNodesWithSpace(List<DatanodeDetails> nodes,
int nodesRequired, long metadataSizeRequired, long dataSizeRequired)
throws SCMException {
List<DatanodeDetails> nodesWithSpace = nodes.stream().filter(d ->
hasEnoughSpace(d, metadataSizeRequired, dataSizeRequired))
hasEnoughSpace(d, metadataSizeRequired, dataSizeRequired, conf))
.collect(Collectors.toList());

if (nodesWithSpace.size() < nodesRequired) {
Expand All @@ -298,7 +299,9 @@ public List<DatanodeDetails> filterNodesWithSpace(List<DatanodeDetails> nodes,
* @return true if we have enough space.
*/
public static boolean hasEnoughSpace(DatanodeDetails datanodeDetails,
long metadataSizeRequired, long dataSizeRequired) {
long metadataSizeRequired,
long dataSizeRequired,
ConfigurationSource conf) {
Preconditions.checkArgument(datanodeDetails instanceof DatanodeInfo);

boolean enoughForData = false;
Expand All @@ -308,7 +311,9 @@ public static boolean hasEnoughSpace(DatanodeDetails datanodeDetails,

if (dataSizeRequired > 0) {
for (StorageReportProto reportProto : datanodeInfo.getStorageReports()) {
if (reportProto.getRemaining() > dataSizeRequired) {
if (VolumeUsage.hasVolumeEnoughSpace(reportProto.getRemaining(),
reportProto.getCommitted(), dataSizeRequired,
reportProto.getFreeSpaceToSpare())) {
enoughForData = true;
break;
}
Expand Down Expand Up @@ -494,7 +499,7 @@ public boolean isValidNode(DatanodeDetails datanodeDetails,
NodeStatus nodeStatus = datanodeInfo.getNodeStatus();
if (nodeStatus.isNodeWritable() &&
(hasEnoughSpace(datanodeInfo, metadataSizeRequired,
dataSizeRequired))) {
dataSizeRequired, conf))) {
LOG.debug("Datanode {} is chosen. Required metadata size is {} and " +
"required data size is {} and NodeStatus is {}",
datanodeDetails, metadataSizeRequired, dataSizeRequired, nodeStatus);
Expand Down
2 changes: 1 addition & 1 deletion ...cm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -932,7 +932,7 @@ private long ratioToBytes(Long nodeCapacity, double utilizationRatio) {
return 0;
}
SCMNodeStat aggregatedStats = new SCMNodeStat(
0, 0, 0);
0, 0, 0, 0, 0);
for (DatanodeUsageInfo node : nodes) {
aggregatedStats.add(node.getScmNodeStat());
}
Expand Down
15 changes: 14 additions & 1 deletion ...er-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/NodeStat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,27 @@ interface NodeStat {
*/
LongMetric getRemaining();

/**
* Get the committed space of the node.
* @return the committed space of the node
*/
LongMetric getCommitted();

/**
* Get a min free space available to spare on the node.
* @return a min free space available to spare
*/
LongMetric getFreeSpaceToSpare();

/**
* Set the total/used/remaining space.
* @param capacity - total space.
* @param used - used space.
* @param remain - remaining space.
*/
@VisibleForTesting
void set(long capacity, long used, long remain);
void set(long capacity, long used, long remain, long committed,
long freeSpaceToSpare);

/**
* Adding of the stat.
Expand Down
16 changes: 10 additions & 6 deletions ...m/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMNodeMetric.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,19 @@ public SCMNodeMetric(SCMNodeStat stat) {
}

/**
* Set the capacity, used and remaining space on a datanode.
* Set the capacity, used, remaining and committed space on a datanode.
*
* @param capacity in bytes
* @param used in bytes
* @param capacity in bytes
* @param used in bytes
* @param remaining in bytes
* @param committed
* @paaram committed in bytes
*/
@VisibleForTesting
public SCMNodeMetric(long capacity, long used, long remaining) {
public SCMNodeMetric(long capacity, long used, long remaining,
long committed, long freeSpaceToSpare) {
this.stat = new SCMNodeStat();
this.stat.set(capacity, used, remaining);
this.stat.set(capacity, used, remaining, committed, freeSpaceToSpare);
}

/**
Expand Down Expand Up @@ -156,7 +159,8 @@ public SCMNodeStat get() {
@Override
public void set(SCMNodeStat value) {
stat.set(value.getCapacity().get(), value.getScmUsed().get(),
value.getRemaining().get());
value.getRemaining().get(), value.getCommitted().get(),
value.getFreeSpaceToSpare().get());
}

/**
Expand Down
Loading