From 3a87a2a6631b5e52e322d72e229e57dd6967aa63 Mon Sep 17 00:00:00 2001 From: Lukas Raska Date: Thu, 17 Nov 2022 20:02:32 +0100 Subject: [PATCH] AdminClient: add getDataUsageInfo API (#1382) Signed-off-by: Lukas Raska --- .../java/io/minio/admin/MinioAdminClient.java | 22 ++++ .../io/minio/admin/messages/AllTierStats.java | 39 +++++++ .../admin/messages/BucketTargetUsageInfo.java | 72 ++++++++++++ .../minio/admin/messages/BucketUsageInfo.java | 109 ++++++++++++++++++ .../minio/admin/messages/DataUsageInfo.java | 97 ++++++++++++++++ .../io/minio/admin/messages/TierStats.java | 51 ++++++++ .../admin/messages/DataUsageInfoTest.java | 54 +++++++++ .../resources/messages/datausageinfo.json | 43 +++++++ build.gradle | 1 + 9 files changed, 488 insertions(+) create mode 100644 adminapi/src/main/java/io/minio/admin/messages/AllTierStats.java create mode 100644 adminapi/src/main/java/io/minio/admin/messages/BucketTargetUsageInfo.java create mode 100644 adminapi/src/main/java/io/minio/admin/messages/BucketUsageInfo.java create mode 100644 adminapi/src/main/java/io/minio/admin/messages/DataUsageInfo.java create mode 100644 adminapi/src/main/java/io/minio/admin/messages/TierStats.java create mode 100644 adminapi/src/test/java/io/minio/admin/messages/DataUsageInfoTest.java create mode 100644 adminapi/src/test/resources/messages/datausageinfo.json diff --git a/adminapi/src/main/java/io/minio/admin/MinioAdminClient.java b/adminapi/src/main/java/io/minio/admin/MinioAdminClient.java index b96bba0af..a2ab37789 100644 --- a/adminapi/src/main/java/io/minio/admin/MinioAdminClient.java +++ b/adminapi/src/main/java/io/minio/admin/MinioAdminClient.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.type.CollectionType; import com.fasterxml.jackson.databind.type.MapType; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Multimap; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -29,6 +30,7 @@ import io.minio.S3Escaper; import io.minio.Signer; import io.minio.Time; +import io.minio.admin.messages.DataUsageInfo; import io.minio.credentials.Credentials; import io.minio.credentials.Provider; import io.minio.credentials.StaticProvider; @@ -74,6 +76,7 @@ private enum Command { REMOVE_CANNED_POLICY("remove-canned-policy"), SET_BUCKET_QUOTA("set-bucket-quota"), GET_BUCKET_QUOTA("get-bucket-quota"), + DATA_USAGE_INFO("datausageinfo"), ADD_UPDATE_REMOVE_GROUP("update-group-members"), GROUP_INFO("group"), LIST_GROUPS("groups"); @@ -92,6 +95,10 @@ public String toString() { private static final MediaType DEFAULT_MEDIA_TYPE = MediaType.parse("application/octet-stream"); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + static { + OBJECT_MAPPER.registerModule(new JavaTimeModule()); + } + private String userAgent = MinioProperties.INSTANCE.getDefaultUserAgent(); private PrintWriter traceStream; @@ -575,6 +582,21 @@ public void removeCannedPolicy(@Nonnull String name) null)) {} } + /** + * Get server/cluster data usage info + * + * @return DataUsageInfo object + * @throws NoSuchAlgorithmException thrown to indicate missing of MD5 or SHA-256 digest library. + * @throws InvalidKeyException thrown to indicate missing of HMAC SHA-256 library. + * @throws IOException thrown to indicate I/O error on MinIO REST operation. + */ + public DataUsageInfo getDataUsageInfo() + throws IOException, NoSuchAlgorithmException, InvalidKeyException { + try (Response response = execute(Method.GET, Command.DATA_USAGE_INFO, null, null)) { + return OBJECT_MAPPER.readValue(response.body().bytes(), DataUsageInfo.class); + } + } + /** * Sets HTTP connect, write and read timeouts. A value of 0 means no timeout, otherwise values * must be between 1 and Integer.MAX_VALUE when converted to milliseconds. diff --git a/adminapi/src/main/java/io/minio/admin/messages/AllTierStats.java b/adminapi/src/main/java/io/minio/admin/messages/AllTierStats.java new file mode 100644 index 000000000..801178186 --- /dev/null +++ b/adminapi/src/main/java/io/minio/admin/messages/AllTierStats.java @@ -0,0 +1,39 @@ +/* + * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, + * (C) 2022 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.minio.admin.messages; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Collections; +import java.util.Map; + +/** + * Collection of per-tier stats across all configured remote tiers + * + * @see data-usage-cache.go + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class AllTierStats { + @JsonProperty("Tiers") + private Map tiers; + + public Map tiers() { + return Collections.unmodifiableMap(this.tiers); + } +} diff --git a/adminapi/src/main/java/io/minio/admin/messages/BucketTargetUsageInfo.java b/adminapi/src/main/java/io/minio/admin/messages/BucketTargetUsageInfo.java new file mode 100644 index 000000000..40dee2728 --- /dev/null +++ b/adminapi/src/main/java/io/minio/admin/messages/BucketTargetUsageInfo.java @@ -0,0 +1,72 @@ +/* + * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, + * (C) 2022 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.minio.admin.messages; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Represents bucket replica stats of the current object APi. + * + * @see data-usage-utils.go + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class BucketTargetUsageInfo { + @JsonProperty("objectsPendingReplicationTotalSize") + private long objectsPendingReplicationTotalSize; + + @JsonProperty("objectsFailedReplicationTotalSize") + private long objectsFailedReplicationTotalSize; + + @JsonProperty("objectsReplicatedTotalSize") + private long objectsReplicatedTotalSize; + + @JsonProperty("objectReplicaTotalSize") + private long objectReplicaTotalSize; + + @JsonProperty("objectsPendingReplicationCount") + private long objectsPendingReplicationCount; + + @JsonProperty("objectsFailedReplicationCount") + private long objectsFailedReplicationCount; + + public long objectsPendingReplicationTotalSize() { + return objectsPendingReplicationTotalSize; + } + + public long objectsFailedReplicationTotalSize() { + return objectsFailedReplicationTotalSize; + } + + public long objectsReplicatedTotalSize() { + return objectsReplicatedTotalSize; + } + + public long objectReplicaTotalSize() { + return objectReplicaTotalSize; + } + + public long objectsPendingReplicationCount() { + return objectsPendingReplicationCount; + } + + public long objectsFailedReplicationCount() { + return objectsFailedReplicationCount; + } +} diff --git a/adminapi/src/main/java/io/minio/admin/messages/BucketUsageInfo.java b/adminapi/src/main/java/io/minio/admin/messages/BucketUsageInfo.java new file mode 100644 index 000000000..91b47ced1 --- /dev/null +++ b/adminapi/src/main/java/io/minio/admin/messages/BucketUsageInfo.java @@ -0,0 +1,109 @@ +/* + * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, + * (C) 2022 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.minio.admin.messages; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Collections; +import java.util.Map; + +/** + * Represents bucket usage stats of the current object APi. + * + * @see data-usage-utils.go + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class BucketUsageInfo { + @JsonProperty("size") + private long size; + + @JsonProperty("objectsPendingReplicationTotalSize") + private long objectsPendingReplicationTotalSize; + + @JsonProperty("objectsFailedReplicationTotalSize") + private long objectsFailedReplicationTotalSize; + + @JsonProperty("objectsReplicatedTotalSize") + private long objectsReplicatedTotalSize; + + @JsonProperty("objectsPendingReplicationCount") + private long objectsPendingReplicationCount; + + @JsonProperty("objectsFailedReplicationCount") + private long objectsFailedReplicationCount; + + @JsonProperty("objectsCount") + private long objectsCount; + + @JsonProperty("objectsSizesHistogram") + private Map objectsSizesHistogram; + + @JsonProperty("versionsCount") + private long versionsCount; + + @JsonProperty("objectReplicaTotalSize") + private long objectReplicaTotalSize; + + @JsonProperty("objectsReplicationInfo") + private Map objectsReplicationInfo; + + public long size() { + return size; + } + + public long objectsPendingReplicationTotalSize() { + return objectsPendingReplicationTotalSize; + } + + public long objectsFailedReplicationTotalSize() { + return objectsFailedReplicationTotalSize; + } + + public long objectsReplicatedTotalSize() { + return objectsReplicatedTotalSize; + } + + public long objectsPendingReplicationCount() { + return objectsPendingReplicationCount; + } + + public long objectsFailedReplicationCount() { + return objectsFailedReplicationCount; + } + + public long objectsCount() { + return objectsCount; + } + + public Map objectsSizesHistogram() { + return Collections.unmodifiableMap(this.objectsSizesHistogram); + } + + public long versionsCount() { + return versionsCount; + } + + public long objectReplicaTotalSize() { + return objectReplicaTotalSize; + } + + public Map objectsReplicationInfo() { + return Collections.unmodifiableMap(this.objectsReplicationInfo); + } +} diff --git a/adminapi/src/main/java/io/minio/admin/messages/DataUsageInfo.java b/adminapi/src/main/java/io/minio/admin/messages/DataUsageInfo.java new file mode 100644 index 000000000..138150cf6 --- /dev/null +++ b/adminapi/src/main/java/io/minio/admin/messages/DataUsageInfo.java @@ -0,0 +1,97 @@ +/* + * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, + * (C) 2022 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.minio.admin.messages; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.time.ZonedDateTime; +import java.util.Collections; +import java.util.Map; + +/** + * Represents data usage stats of the current object APi. + * + * @see data-usage-utils.go + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class DataUsageInfo { + + @JsonProperty("lastUpdate") + private ZonedDateTime lastUpdate; + + @JsonProperty("objectsCount") + private long objectsCount; + + @JsonProperty("versionsCount") + private long versionsCount; + + @JsonProperty("objectsTotalSize") + private long objectsTotalSize; + + @JsonProperty("objectsReplicationInfo") + private Map objectsReplicationInfo; + + @JsonProperty("bucketsCount") + private long bucketsCount; + + @JsonProperty("bucketsUsageInfo") + private Map bucketsUsageInfo; + + @JsonProperty("bucketsSizes") + private Map bucketsSizes; + + @JsonProperty("tierStats") + private AllTierStats tierStats; + + public ZonedDateTime lastUpdate() { + return lastUpdate; + } + + public long objectsCount() { + return objectsCount; + } + + public long versionsCount() { + return versionsCount; + } + + public long objectsTotalSize() { + return objectsTotalSize; + } + + public Map objectsReplicationInfo() { + return Collections.unmodifiableMap(this.objectsReplicationInfo); + } + + public long bucketsCount() { + return bucketsCount; + } + + public Map bucketsUsageInfo() { + return Collections.unmodifiableMap(this.bucketsUsageInfo); + } + + public Map bucketsSizes() { + return bucketsSizes; + } + + public AllTierStats tierStats() { + return tierStats; + } +} diff --git a/adminapi/src/main/java/io/minio/admin/messages/TierStats.java b/adminapi/src/main/java/io/minio/admin/messages/TierStats.java new file mode 100644 index 000000000..034fa40ca --- /dev/null +++ b/adminapi/src/main/java/io/minio/admin/messages/TierStats.java @@ -0,0 +1,51 @@ +/* + * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, + * (C) 2022 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.minio.admin.messages; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Per-tier stats of a remote tier. + * + * @see data-usage-cache.go + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class TierStats { + @JsonProperty("TotalSize") + private long totalSize; + + @JsonProperty("NumVersions") + private int numVersions; + + @JsonProperty("NumObjects") + private int numObjects; + + public long totalSize() { + return totalSize; + } + + public int numVersions() { + return numVersions; + } + + public int numObjects() { + return numObjects; + } +} diff --git a/adminapi/src/test/java/io/minio/admin/messages/DataUsageInfoTest.java b/adminapi/src/test/java/io/minio/admin/messages/DataUsageInfoTest.java new file mode 100644 index 000000000..dfe01b7a0 --- /dev/null +++ b/adminapi/src/test/java/io/minio/admin/messages/DataUsageInfoTest.java @@ -0,0 +1,54 @@ +/* + * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, + * (C) 2022 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.minio.admin.messages; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import java.io.File; +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import org.junit.Assert; +import org.junit.Test; + +public class DataUsageInfoTest { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + static { + OBJECT_MAPPER.registerModule(new JavaTimeModule()); + } + + @Test + public void deserializeTest() throws IOException { + + DataUsageInfo info = + OBJECT_MAPPER.readValue( + new File( + getClass().getClassLoader().getResource("messages/datausageinfo.json").getFile()), + DataUsageInfo.class); + Assert.assertNotNull(info.lastUpdate()); + Assert.assertEquals( + 15, LocalDateTime.ofInstant(info.lastUpdate().toInstant(), ZoneOffset.UTC).getHour()); + + Assert.assertEquals(1, info.bucketsCount()); + Assert.assertTrue(info.bucketsUsageInfo().containsKey("tier-bucket")); + Assert.assertTrue(info.tierStats().tiers().containsKey("STANDARD")); + + Assert.assertEquals(7155L, (long) info.bucketsSizes().get("tier-bucket")); + } +} diff --git a/adminapi/src/test/resources/messages/datausageinfo.json b/adminapi/src/test/resources/messages/datausageinfo.json new file mode 100644 index 000000000..1ea3ff5d8 --- /dev/null +++ b/adminapi/src/test/resources/messages/datausageinfo.json @@ -0,0 +1,43 @@ +{ + "bucketsCount": 1, + "bucketsSizes": { + "tier-bucket": 7155 + }, + "bucketsUsageInfo": { + "tier-bucket": { + "objectReplicaTotalSize": 0, + "objectsCount": 1, + "objectsFailedReplicationCount": 0, + "objectsFailedReplicationTotalSize": 0, + "objectsPendingReplicationCount": 0, + "objectsPendingReplicationTotalSize": 0, + "objectsReplicatedTotalSize": 0, + "objectsReplicationInfo": {}, + "objectsSizesHistogram": { + "BETWEEN_1024_B_AND_1_MB": 1, + "BETWEEN_10_MB_AND_64_MB": 0, + "BETWEEN_128_MB_AND_512_MB": 0, + "BETWEEN_1_MB_AND_10_MB": 0, + "BETWEEN_64_MB_AND_128_MB": 0, + "GREATER_THAN_512_MB": 0, + "LESS_THAN_1024_B": 0 + }, + "size": 7155, + "versionsCount": 1 + } + }, + "lastUpdate": "2022-11-08T16:15:56.277821865+01:00", + "objectsCount": 57, + "objectsReplicationInfo": null, + "objectsTotalSize": 4296263795, + "tierStats": { + "Tiers": { + "STANDARD": { + "NumObjects": 2, + "NumVersions": 2, + "TotalSize": 14310 + } + } + }, + "versionsCount": 1 +} diff --git a/build.gradle b/build.gradle index 47b8e493c..9a6276722 100644 --- a/build.gradle +++ b/build.gradle @@ -242,6 +242,7 @@ project(':adminapi') { dependencies { compile project(':api') + compile "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.13.4" testImplementation project(':api') }