From 325df779e798be7854259c285c3e72d5a276c1d0 Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Fri, 2 May 2025 14:54:14 -0400 Subject: [PATCH 01/22] Initial implementation of thread dumps --- smoketest.bash | 2 +- .../java/io/cryostat/ConfigProperties.java | 1 + .../io/cryostat/diagnostic/Diagnostics.java | 262 ++++++++++++++++++ .../java/io/cryostat/targets/AgentClient.java | 4 + .../io/cryostat/targets/AgentConnection.java | 1 - src/main/resources/application-dev.properties | 2 +- src/main/resources/application.properties | 1 + .../cryostat/resources/S3StorageResource.java | 3 +- 8 files changed, 272 insertions(+), 4 deletions(-) diff --git a/smoketest.bash b/smoketest.bash index 1a38ca329..7da476fc1 100755 --- a/smoketest.bash +++ b/smoketest.bash @@ -17,7 +17,7 @@ PULL_IMAGES=${PULL_IMAGES:-true} KEEP_VOLUMES=${KEEP_VOLUMES:-false} OPEN_TABS=${OPEN_TABS:-false} -PRECREATE_BUCKETS=${PRECREATE_BUCKETS:-archivedrecordings,archivedreports,eventtemplates,probes} +PRECREATE_BUCKETS=${PRECREATE_BUCKETS:-archivedrecordings,archivedreports,eventtemplates,probes,threaddumps} LOG_LEVEL=0 CRYOSTAT_HTTP_HOST=${CRYOSTAT_HTTP_HOST:-cryostat} diff --git a/src/main/java/io/cryostat/ConfigProperties.java b/src/main/java/io/cryostat/ConfigProperties.java index f78921829..75d909cc6 100644 --- a/src/main/java/io/cryostat/ConfigProperties.java +++ b/src/main/java/io/cryostat/ConfigProperties.java @@ -21,6 +21,7 @@ public class ConfigProperties { "storage.buckets.event-templates.name"; public static final String AWS_BUCKET_NAME_PROBE_TEMPLATES = "storage.buckets.probe-templates.name"; + public static final String AWS_BUCKET_NAME_THREAD_DUMPS = "storage.buckets.thread-dumps.name"; public static final String CONTAINERS_POLL_PERIOD = "cryostat.discovery.containers.poll-period"; public static final String CONTAINERS_REQUEST_TIMEOUT = diff --git a/src/main/java/io/cryostat/diagnostic/Diagnostics.java b/src/main/java/io/cryostat/diagnostic/Diagnostics.java index 1ae118f41..2680a0b02 100644 --- a/src/main/java/io/cryostat/diagnostic/Diagnostics.java +++ b/src/main/java/io/cryostat/diagnostic/Diagnostics.java @@ -15,20 +15,272 @@ */ package io.cryostat.diagnostic; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; + +import io.cryostat.ConfigProperties; +import io.cryostat.Producers; +import io.cryostat.StorageBuckets; import io.cryostat.targets.Target; import io.cryostat.targets.TargetConnectionManager; +import io.cryostat.util.HttpMimeType; +import io.quarkus.runtime.StartupEvent; import io.smallrye.common.annotation.Blocking; import jakarta.annotation.security.RolesAllowed; +import jakarta.enterprise.event.Observes; import jakarta.inject.Inject; +import jakarta.inject.Named; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.NotFoundException; import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; +import jakarta.ws.rs.core.HttpHeaders; +import jakarta.ws.rs.core.MediaType; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.jboss.logging.Logger; import org.jboss.resteasy.reactive.RestPath; +import org.jboss.resteasy.reactive.RestQuery; +import org.jboss.resteasy.reactive.RestResponse; +import org.jboss.resteasy.reactive.RestResponse.ResponseBuilder; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.services.s3.model.Tag; +import software.amazon.awssdk.services.s3.model.Tagging; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; +import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; @Path("/api/beta/diagnostics/targets/{targetId}") public class Diagnostics { @Inject TargetConnectionManager targetConnectionManager; + @Inject StorageBuckets buckets; + @Inject S3Client storage; + @Inject S3Presigner presigner; + @Inject Logger log; + + @Inject + @Named(Producers.BASE64_URL) + Base64 base64Url; + + private static final String DUMP_THREADS = "threadPrint"; + private static final String DIAGNOSTIC_BEAN_NAME = "com.sun.management:type=DiagnosticCommand"; + + @ConfigProperty(name = ConfigProperties.AWS_BUCKET_NAME_THREAD_DUMPS) + String bucket; + + @ConfigProperty(name = ConfigProperties.STORAGE_PRESIGNED_DOWNLOADS_ENABLED) + boolean presignedDownloadsEnabled; + + @ConfigProperty(name = ConfigProperties.STORAGE_EXT_URL) + Optional externalStorageUrl; + + void onStart(@Observes StartupEvent evt) { + buckets.createIfNecessary(bucket); + } + + @Path("/threaddump") + @RolesAllowed("write") + @Blocking + @POST + public void threadDump(@RestPath long targetId) { + Object[] params = new Object[1]; + String[] signature = new String[] {String[].class.getName()}; + targetConnectionManager.executeConnectedTask( + Target.getTargetById(targetId), + conn -> { + String content = + conn.invokeMBeanOperation( + DIAGNOSTIC_BEAN_NAME, + DUMP_THREADS, + params, + signature, + String.class); + addThreadDump(content, Target.getTargetById(targetId).jvmId); + return content; + }); + } + + @DELETE + @Blocking + @Path("/{threadDumpId}") + @RolesAllowed("write") + public void deleteThreadDump(@RestPath String threadDumpId) { + try { + storage.headObject( + HeadObjectRequest.builder().bucket(bucket).key(threadDumpId).build()); + } catch (NoSuchKeyException e) { + throw new NotFoundException(e); + } + storage.deleteObject( + DeleteObjectRequest.builder().bucket(bucket).key(threadDumpId).build()); + } + + @Path("/threaddump/download/{encodedKey}") + @RolesAllowed("read") + @Blocking + @GET + public RestResponse handleStorageDownload( + @RestPath String encodedKey, @RestQuery String query) throws URISyntaxException { + Pair decodedKey = decodedKey(encodedKey); + String key = threadDumpKey(decodedKey); + + storage.headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build()) + .sdkHttpResponse(); + + if (!presignedDownloadsEnabled) { + return ResponseBuilder.ok() + .header( + HttpHeaders.CONTENT_DISPOSITION, + String.format("attachment; filename=\"%s\"", decodedKey.getValue())) + .header(HttpHeaders.CONTENT_TYPE, HttpMimeType.OCTET_STREAM.mime()) + .entity(getThreadDumpStream(encodedKey)) + .build(); + } + + log.tracev("Handling presigned download request for {0}", decodedKey); + GetObjectRequest getRequest = GetObjectRequest.builder().bucket(bucket).key(key).build(); + GetObjectPresignRequest presignRequest = + GetObjectPresignRequest.builder() + .signatureDuration(Duration.ofMinutes(1)) + .getObjectRequest(getRequest) + .build(); + PresignedGetObjectRequest presignedRequest = presigner.presignGetObject(presignRequest); + URI uri = presignedRequest.url().toURI(); + if (externalStorageUrl.isPresent()) { + String extUrl = externalStorageUrl.get(); + if (StringUtils.isNotBlank(extUrl)) { + URI extUri = new URI(extUrl); + uri = + new URI( + extUri.getScheme(), + extUri.getAuthority(), + URI.create(String.format("%s/%s", extUri.getPath(), uri.getPath())) + .normalize() + .getPath(), + uri.getQuery(), + uri.getFragment()); + } + } + ResponseBuilder response = + ResponseBuilder.create(RestResponse.Status.PERMANENT_REDIRECT); + if (StringUtils.isNotBlank(query)) { + response = + response.header( + HttpHeaders.CONTENT_DISPOSITION, + String.format( + "attachment; filename=\"%s\"", + new String(base64Url.decode(query), StandardCharsets.UTF_8))); + } + return response.location(uri).build(); + } + + public InputStream getThreadDumpStream(String jvmId, String threadDumpID) { + return getThreadDumpStream(encodedKey(jvmId, threadDumpID)); + } + + public InputStream getThreadDumpStream(String encodedKey) { + String key = new String(base64Url.decode(encodedKey), StandardCharsets.UTF_8); + + GetObjectRequest getRequest = GetObjectRequest.builder().bucket(bucket).key(key).build(); + + return storage.getObject(getRequest); + } + + public Pair decodedKey(String encodedKey) { + String key = new String(base64Url.decode(encodedKey), StandardCharsets.UTF_8); + String[] parts = key.split("/"); + if (parts.length != 2) { + throw new IllegalArgumentException(); + } + return Pair.of(parts[0], parts[1]); + } + + public String encodedKey(String jvmId, String filename) { + Objects.requireNonNull(jvmId); + Objects.requireNonNull(filename); + return base64Url.encodeAsString( + (threadDumpKey(jvmId, filename)).getBytes(StandardCharsets.UTF_8)); + } + + public String threadDumpKey(String jvmId, String filename) { + return (jvmId + "/" + filename).strip(); + } + + public String threadDumpKey(Pair pair) { + return threadDumpKey(pair.getKey(), pair.getValue()); + } + + public ThreadDump addThreadDump(String content, String jvmId) { + String uuid = UUID.randomUUID().toString(); + storage.putObject( + PutObjectRequest.builder() + .bucket(bucket) + .key(uuid) + .contentType(MediaType.TEXT_PLAIN) + .tagging(createTagging(jvmId, uuid)) + .build(), + RequestBody.fromString(content)); + return new ThreadDump(content, jvmId, downloadUrl(jvmId, uuid), uuid); + } + + private Tagging createTagging(String jvmId, String uuid) { + var map = Map.of("jvmId", jvmId, "uuid", uuid); + var tags = new ArrayList(); + tags.addAll( + map.entrySet().stream() + .map( + e -> + Tag.builder() + .key( + base64Url.encodeAsString( + e.getKey() + .getBytes( + StandardCharsets + .UTF_8))) + .value( + base64Url.encodeAsString( + e.getValue() + .getBytes( + StandardCharsets + .UTF_8))) + .build()) + .toList()); + return Tagging.builder().tagSet(tags).build(); + } + + public List listThreadDumps(String jvmId) { + var builder = ListObjectsV2Request.builder().bucket(bucket); + if (StringUtils.isNotBlank(jvmId)) { + builder = builder.prefix(jvmId); + } + return storage.listObjectsV2(builder.build()).contents().stream().toList(); + } + + public String downloadUrl(String jvmId, String filename) { + return String.format("/threaddump/download/%s", encodedKey(jvmId, filename)); + } @Path("/gc") @RolesAllowed("write") @@ -41,4 +293,14 @@ public void gc(@RestPath long targetId) { conn.invokeMBeanOperation( "java.lang:type=Memory", "gc", null, null, Void.class)); } + + public record ThreadDump(String content, String jvmId, String downloadUrl, String uuid) { + + public ThreadDump { + Objects.requireNonNull(content); + Objects.requireNonNull(jvmId); + Objects.requireNonNull(downloadUrl); + Objects.requireNonNull(uuid); + } + } } diff --git a/src/main/java/io/cryostat/targets/AgentClient.java b/src/main/java/io/cryostat/targets/AgentClient.java index dd5b0e22d..7fd1796e2 100644 --- a/src/main/java/io/cryostat/targets/AgentClient.java +++ b/src/main/java/io/cryostat/targets/AgentClient.java @@ -112,6 +112,7 @@ Uni mbeanMetrics() { .map(Unchecked.function(s -> mapper.readValue(s, MBeanMetrics.class))); } + @SuppressWarnings("unchecked") Uni invokeMBeanOperation( String beanName, String operation, @@ -150,6 +151,9 @@ Uni invokeMBeanOperation( .map(HttpResponse::bodyAsBuffer) .map( buff -> { + if (returnType.equals(String.class)) { + return (T) buff.toString(); + } // TODO implement conditional handling based on expected returnType return null; }); diff --git a/src/main/java/io/cryostat/targets/AgentConnection.java b/src/main/java/io/cryostat/targets/AgentConnection.java index 09e1805f0..e52326699 100644 --- a/src/main/java/io/cryostat/targets/AgentConnection.java +++ b/src/main/java/io/cryostat/targets/AgentConnection.java @@ -109,7 +109,6 @@ public JvmIdentifier getJvmIdentifier() throws IDException, IOException { } } - @Override public T invokeMBeanOperation( String beanName, String operation, diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index fcdd73337..f7890abe8 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -37,7 +37,7 @@ quarkus.datasource.devservices.db-name=quarkus # !!! quarkus.s3.devservices.enabled=true -quarkus.s3.devservices.buckets=archivedrecordings,archivedreports,eventtemplates,probes +quarkus.s3.devservices.buckets=archivedrecordings,archivedreports,eventtemplates,probes,threaddumps # FIXME the following overrides should not be required, but currently seem to help with testcontainers reliability quarkus.aws.devservices.localstack.image-name=quay.io/hazelcast_cloud/localstack:4.1.1 quarkus.aws.devservices.localstack.container-properties.START_WEB=0 diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 8b662bfd7..7de09d33f 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -112,6 +112,7 @@ storage.buckets.creation-retry.period=10s storage.buckets.archives.name=archivedrecordings storage.buckets.event-templates.name=eventtemplates storage.buckets.probe-templates.name=probes +storage.buckets.thread-dumps.name=threaddumps quarkus.quinoa.build-dir=dist quarkus.quinoa.enable-spa-routing=true diff --git a/src/test/java/io/cryostat/resources/S3StorageResource.java b/src/test/java/io/cryostat/resources/S3StorageResource.java index c07e97654..76f6b83c8 100644 --- a/src/test/java/io/cryostat/resources/S3StorageResource.java +++ b/src/test/java/io/cryostat/resources/S3StorageResource.java @@ -39,7 +39,8 @@ public class S3StorageResource "REST_ENCRYPTION_ENABLE", "1", "CRYOSTAT_ACCESS_KEY", "access_key", "CRYOSTAT_SECRET_KEY", "secret_key", - "CRYOSTAT_BUCKETS", "archivedrecordings,archivedreports,eventtemplates,probes"); + "CRYOSTAT_BUCKETS", + "archivedrecordings,archivedreports,eventtemplates,probes,threaddumps"); private static final Logger logger = Logger.getLogger(S3StorageResource.class); private Optional containerNetworkId; private GenericContainer container; From e74a9d52e10eaa04fe22d53682bd9f40495a48cd Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Fri, 2 May 2025 15:02:16 -0400 Subject: [PATCH 02/22] Support other thread dump format, sanity check format query parameter --- src/main/java/io/cryostat/diagnostic/Diagnostics.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/cryostat/diagnostic/Diagnostics.java b/src/main/java/io/cryostat/diagnostic/Diagnostics.java index 2680a0b02..ebac42396 100644 --- a/src/main/java/io/cryostat/diagnostic/Diagnostics.java +++ b/src/main/java/io/cryostat/diagnostic/Diagnostics.java @@ -40,6 +40,7 @@ import jakarta.enterprise.event.Observes; import jakarta.inject.Inject; import jakarta.inject.Named; +import jakarta.ws.rs.BadRequestException; import jakarta.ws.rs.DELETE; import jakarta.ws.rs.GET; import jakarta.ws.rs.NotFoundException; @@ -85,6 +86,7 @@ public class Diagnostics { Base64 base64Url; private static final String DUMP_THREADS = "threadPrint"; + private static final String DUMP_THREADS_TO_FIlE = "threadDumpToFile"; private static final String DIAGNOSTIC_BEAN_NAME = "com.sun.management:type=DiagnosticCommand"; @ConfigProperty(name = ConfigProperties.AWS_BUCKET_NAME_THREAD_DUMPS) @@ -104,7 +106,10 @@ void onStart(@Observes StartupEvent evt) { @RolesAllowed("write") @Blocking @POST - public void threadDump(@RestPath long targetId) { + public void threadDump(@RestPath long targetId, @RestQuery String format) { + if (!(format.equals(DUMP_THREADS) || format.equals(DUMP_THREADS_TO_FIlE))) { + throw new BadRequestException(); + } Object[] params = new Object[1]; String[] signature = new String[] {String[].class.getName()}; targetConnectionManager.executeConnectedTask( @@ -113,7 +118,7 @@ public void threadDump(@RestPath long targetId) { String content = conn.invokeMBeanOperation( DIAGNOSTIC_BEAN_NAME, - DUMP_THREADS, + format, params, signature, String.class); From 368fa466d1c0d58721636c65f459dd175db5a34c Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Fri, 30 May 2025 15:00:51 -0400 Subject: [PATCH 03/22] refactor, long-running api handling --- .../io/cryostat/diagnostic/Diagnostics.java | 161 +++--------- .../diagnostic/DiagnosticsHelper.java | 244 ++++++++++++++++++ .../LongRunningRequestGenerator.java | 43 +++ 3 files changed, 318 insertions(+), 130 deletions(-) create mode 100644 src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java diff --git a/src/main/java/io/cryostat/diagnostic/Diagnostics.java b/src/main/java/io/cryostat/diagnostic/Diagnostics.java index ebac42396..acfdcc5a3 100644 --- a/src/main/java/io/cryostat/diagnostic/Diagnostics.java +++ b/src/main/java/io/cryostat/diagnostic/Diagnostics.java @@ -15,39 +15,35 @@ */ package io.cryostat.diagnostic; -import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.time.Duration; -import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.UUID; import io.cryostat.ConfigProperties; import io.cryostat.Producers; -import io.cryostat.StorageBuckets; +import io.cryostat.recordings.LongRunningRequestGenerator; +import io.cryostat.recordings.LongRunningRequestGenerator.ThreadDumpRequest; import io.cryostat.targets.Target; import io.cryostat.targets.TargetConnectionManager; import io.cryostat.util.HttpMimeType; -import io.quarkus.runtime.StartupEvent; import io.smallrye.common.annotation.Blocking; +import io.vertx.core.http.HttpServerResponse; +import io.vertx.mutiny.core.eventbus.EventBus; import jakarta.annotation.security.RolesAllowed; -import jakarta.enterprise.event.Observes; import jakarta.inject.Inject; import jakarta.inject.Named; -import jakarta.ws.rs.BadRequestException; import jakarta.ws.rs.DELETE; import jakarta.ws.rs.GET; import jakarta.ws.rs.NotFoundException; import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import jakarta.ws.rs.core.HttpHeaders; -import jakarta.ws.rs.core.MediaType; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -57,17 +53,11 @@ import org.jboss.resteasy.reactive.RestQuery; import org.jboss.resteasy.reactive.RestResponse; import org.jboss.resteasy.reactive.RestResponse.ResponseBuilder; -import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.HeadObjectRequest; -import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.S3Object; -import software.amazon.awssdk.services.s3.model.Tag; -import software.amazon.awssdk.services.s3.model.Tagging; import software.amazon.awssdk.services.s3.presigner.S3Presigner; import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; @@ -76,19 +66,15 @@ public class Diagnostics { @Inject TargetConnectionManager targetConnectionManager; - @Inject StorageBuckets buckets; @Inject S3Client storage; @Inject S3Presigner presigner; @Inject Logger log; + @Inject LongRunningRequestGenerator generator; @Inject @Named(Producers.BASE64_URL) Base64 base64Url; - private static final String DUMP_THREADS = "threadPrint"; - private static final String DUMP_THREADS_TO_FIlE = "threadDumpToFile"; - private static final String DIAGNOSTIC_BEAN_NAME = "com.sun.management:type=DiagnosticCommand"; - @ConfigProperty(name = ConfigProperties.AWS_BUCKET_NAME_THREAD_DUMPS) String bucket; @@ -98,33 +84,34 @@ public class Diagnostics { @ConfigProperty(name = ConfigProperties.STORAGE_EXT_URL) Optional externalStorageUrl; - void onStart(@Observes StartupEvent evt) { - buckets.createIfNecessary(bucket); - } + @Inject EventBus bus; + @Inject DiagnosticsHelper helper; @Path("/threaddump") @RolesAllowed("write") @Blocking @POST - public void threadDump(@RestPath long targetId, @RestQuery String format) { - if (!(format.equals(DUMP_THREADS) || format.equals(DUMP_THREADS_TO_FIlE))) { - throw new BadRequestException(); - } - Object[] params = new Object[1]; - String[] signature = new String[] {String[].class.getName()}; - targetConnectionManager.executeConnectedTask( - Target.getTargetById(targetId), - conn -> { - String content = - conn.invokeMBeanOperation( - DIAGNOSTIC_BEAN_NAME, - format, - params, - signature, - String.class); - addThreadDump(content, Target.getTargetById(targetId).jvmId); - return content; - }); + public String threadDump( + HttpServerResponse response, @RestPath long targetId, @RestQuery String format) { + log.trace("Creating new thread dump request"); + ThreadDumpRequest request = + new ThreadDumpRequest( + UUID.randomUUID().toString(), Long.toString(targetId), format); + response.endHandler( + (e) -> + bus.publish( + LongRunningRequestGenerator.GRAFANA_ARCHIVE_REQUEST_ADDRESS, + request)); + return request.id(); + } + + @Path("/threaddump") + @RolesAllowed("read") + @Blocking + @GET + public List getThreadDumps(@RestPath long targetId) { + log.trace("Fetching thread dumps"); + return helper.getThreadDumps(bucket); } @DELETE @@ -148,8 +135,8 @@ public void deleteThreadDump(@RestPath String threadDumpId) { @GET public RestResponse handleStorageDownload( @RestPath String encodedKey, @RestQuery String query) throws URISyntaxException { - Pair decodedKey = decodedKey(encodedKey); - String key = threadDumpKey(decodedKey); + Pair decodedKey = helper.decodedKey(encodedKey); + String key = helper.threadDumpKey(decodedKey); storage.headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build()) .sdkHttpResponse(); @@ -160,7 +147,7 @@ public RestResponse handleStorageDownload( HttpHeaders.CONTENT_DISPOSITION, String.format("attachment; filename=\"%s\"", decodedKey.getValue())) .header(HttpHeaders.CONTENT_TYPE, HttpMimeType.OCTET_STREAM.mime()) - .entity(getThreadDumpStream(encodedKey)) + .entity(helper.getThreadDumpStream(encodedKey)) .build(); } @@ -201,92 +188,6 @@ public RestResponse handleStorageDownload( return response.location(uri).build(); } - public InputStream getThreadDumpStream(String jvmId, String threadDumpID) { - return getThreadDumpStream(encodedKey(jvmId, threadDumpID)); - } - - public InputStream getThreadDumpStream(String encodedKey) { - String key = new String(base64Url.decode(encodedKey), StandardCharsets.UTF_8); - - GetObjectRequest getRequest = GetObjectRequest.builder().bucket(bucket).key(key).build(); - - return storage.getObject(getRequest); - } - - public Pair decodedKey(String encodedKey) { - String key = new String(base64Url.decode(encodedKey), StandardCharsets.UTF_8); - String[] parts = key.split("/"); - if (parts.length != 2) { - throw new IllegalArgumentException(); - } - return Pair.of(parts[0], parts[1]); - } - - public String encodedKey(String jvmId, String filename) { - Objects.requireNonNull(jvmId); - Objects.requireNonNull(filename); - return base64Url.encodeAsString( - (threadDumpKey(jvmId, filename)).getBytes(StandardCharsets.UTF_8)); - } - - public String threadDumpKey(String jvmId, String filename) { - return (jvmId + "/" + filename).strip(); - } - - public String threadDumpKey(Pair pair) { - return threadDumpKey(pair.getKey(), pair.getValue()); - } - - public ThreadDump addThreadDump(String content, String jvmId) { - String uuid = UUID.randomUUID().toString(); - storage.putObject( - PutObjectRequest.builder() - .bucket(bucket) - .key(uuid) - .contentType(MediaType.TEXT_PLAIN) - .tagging(createTagging(jvmId, uuid)) - .build(), - RequestBody.fromString(content)); - return new ThreadDump(content, jvmId, downloadUrl(jvmId, uuid), uuid); - } - - private Tagging createTagging(String jvmId, String uuid) { - var map = Map.of("jvmId", jvmId, "uuid", uuid); - var tags = new ArrayList(); - tags.addAll( - map.entrySet().stream() - .map( - e -> - Tag.builder() - .key( - base64Url.encodeAsString( - e.getKey() - .getBytes( - StandardCharsets - .UTF_8))) - .value( - base64Url.encodeAsString( - e.getValue() - .getBytes( - StandardCharsets - .UTF_8))) - .build()) - .toList()); - return Tagging.builder().tagSet(tags).build(); - } - - public List listThreadDumps(String jvmId) { - var builder = ListObjectsV2Request.builder().bucket(bucket); - if (StringUtils.isNotBlank(jvmId)) { - builder = builder.prefix(jvmId); - } - return storage.listObjectsV2(builder.build()).contents().stream().toList(); - } - - public String downloadUrl(String jvmId, String filename) { - return String.format("/threaddump/download/%s", encodedKey(jvmId, filename)); - } - @Path("/gc") @RolesAllowed("write") @Blocking diff --git a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java new file mode 100644 index 000000000..a5493757b --- /dev/null +++ b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java @@ -0,0 +1,244 @@ +/* + * Copyright The Cryostat Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cryostat.diagnostic; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; + +import io.cryostat.ConfigProperties; +import io.cryostat.Producers; +import io.cryostat.StorageBuckets; +import io.cryostat.diagnostic.Diagnostics.ThreadDump; +import io.cryostat.targets.Target; +import io.cryostat.targets.TargetConnectionManager; + +import io.quarkus.runtime.StartupEvent; +import io.vertx.mutiny.core.eventbus.EventBus; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Observes; +import jakarta.inject.Inject; +import jakarta.inject.Named; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.core.MediaType; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.jboss.logging.Logger; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectTaggingRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.services.s3.model.Tag; +import software.amazon.awssdk.services.s3.model.Tagging; + +@ApplicationScoped +public class DiagnosticsHelper { + + @ConfigProperty(name = ConfigProperties.AWS_BUCKET_NAME_THREAD_DUMPS) + String bucket; + + @Inject + @Named(Producers.BASE64_URL) + Base64 base64Url; + + @Inject S3Client storage; + @Inject Logger log; + + private static final String DUMP_THREADS = "threadPrint"; + private static final String DUMP_THREADS_TO_FIlE = "threadDumpToFile"; + private static final String DIAGNOSTIC_BEAN_NAME = "com.sun.management:type=DiagnosticCommand"; + static final String THREAD_DUMP_REQUESTED = "ThreadDumpRequested"; + static final String THREAD_DUMP_DELETED = "ThreadDumpDeleted"; + + @Inject EventBus bus; + @Inject TargetConnectionManager targetConnectionManager; + @Inject StorageBuckets buckets; + + void onStart(@Observes StartupEvent evt) { + buckets.createIfNecessary(bucket); + } + + public ThreadDump dumpThreads(String format, long targetId) { + if (!(format.equals(DUMP_THREADS) || format.equals(DUMP_THREADS_TO_FIlE))) { + throw new BadRequestException(); + } + Object[] params = new Object[1]; + String[] signature = new String[] {String[].class.getName()}; + return targetConnectionManager.executeConnectedTask( + Target.getTargetById(targetId), + conn -> { + String content = + conn.invokeMBeanOperation( + DIAGNOSTIC_BEAN_NAME, format, params, signature, String.class); + return addThreadDump(content, Target.getTargetById(targetId).jvmId); + }); + } + + public List getThreadDumps(String jvmId) { + return listThreadDumps(jvmId).stream() + .map( + item -> { + try { + return convertObject(item); + } catch (Exception e) { + log.error(e); + return null; + } + }) + .toList(); + } + + private ThreadDump convertObject(S3Object object) throws Exception { + var req = GetObjectTaggingRequest.builder().bucket(bucket).key(object.key()).build(); + var tagging = storage.getObjectTagging(req); + var list = tagging.tagSet(); + if (!tagging.hasTagSet() || list.isEmpty()) { + throw new Exception("No metadata found"); + } + var decodedList = new ArrayList>(); + list.forEach( + t -> { + var encodedKey = t.key(); + var decodedKey = + new String(base64Url.decode(encodedKey), StandardCharsets.UTF_8).trim(); + var encodedValue = t.value(); + var decodedValue = + new String(base64Url.decode(encodedValue), StandardCharsets.UTF_8) + .trim(); + decodedList.add(Pair.of(decodedKey, decodedValue)); + }); + var jvmId = + decodedList.stream() + .filter(t -> t.getKey().equals("jvmId")) + .map(Pair::getValue) + .findFirst() + .orElseThrow(); + var uuid = + decodedList.stream() + .filter(t -> t.getKey().equals("uuid")) + .map(Pair::getValue) + .findFirst() + .orElseThrow(); + // content, jvmid, downloadurl, uuid + return new ThreadDump(getThreadDumpContent(uuid), jvmId, downloadUrl(jvmId, uuid), uuid); + } + + public String getThreadDumpContent(String uuid) throws IOException { + InputStream is = getModel(uuid); + return new String(is.readAllBytes(), StandardCharsets.UTF_8); + } + + private InputStream getModel(String name) { + var req = GetObjectRequest.builder().bucket(bucket).key(name).build(); + return storage.getObject(req); + } + + public ThreadDump addThreadDump(String content, String jvmId) { + String uuid = UUID.randomUUID().toString(); + storage.putObject( + PutObjectRequest.builder() + .bucket(bucket) + .key(uuid) + .contentType(MediaType.TEXT_PLAIN) + .tagging(createTagging(jvmId, uuid)) + .build(), + RequestBody.fromString(content)); + return new ThreadDump(content, jvmId, downloadUrl(jvmId, uuid), uuid); + } + + private Tagging createTagging(String jvmId, String uuid) { + var map = Map.of("jvmId", jvmId, "uuid", uuid); + var tags = new ArrayList(); + tags.addAll( + map.entrySet().stream() + .map( + e -> + Tag.builder() + .key( + base64Url.encodeAsString( + e.getKey() + .getBytes( + StandardCharsets + .UTF_8))) + .value( + base64Url.encodeAsString( + e.getValue() + .getBytes( + StandardCharsets + .UTF_8))) + .build()) + .toList()); + return Tagging.builder().tagSet(tags).build(); + } + + public String downloadUrl(String jvmId, String filename) { + return String.format("/threaddump/download/%s", encodedKey(jvmId, filename)); + } + + public String encodedKey(String jvmId, String filename) { + Objects.requireNonNull(jvmId); + Objects.requireNonNull(filename); + return base64Url.encodeAsString( + (threadDumpKey(jvmId, filename)).getBytes(StandardCharsets.UTF_8)); + } + + public String threadDumpKey(String jvmId, String filename) { + return (jvmId + "/" + filename).strip(); + } + + public String threadDumpKey(Pair pair) { + return threadDumpKey(pair.getKey(), pair.getValue()); + } + + public InputStream getThreadDumpStream(String jvmId, String threadDumpID) { + return getThreadDumpStream(encodedKey(jvmId, threadDumpID)); + } + + public InputStream getThreadDumpStream(String encodedKey) { + String key = new String(base64Url.decode(encodedKey), StandardCharsets.UTF_8); + + GetObjectRequest getRequest = GetObjectRequest.builder().bucket(bucket).key(key).build(); + + return storage.getObject(getRequest); + } + + public Pair decodedKey(String encodedKey) { + String key = new String(base64Url.decode(encodedKey), StandardCharsets.UTF_8); + String[] parts = key.split("/"); + if (parts.length != 2) { + throw new IllegalArgumentException(); + } + return Pair.of(parts[0], parts[1]); + } + + public List listThreadDumps(String jvmId) { + var builder = ListObjectsV2Request.builder().bucket(bucket); + if (StringUtils.isNotBlank(jvmId)) { + builder = builder.prefix(jvmId); + } + return storage.listObjectsV2(builder.build()).contents().stream().toList(); + } +} diff --git a/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java b/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java index 04d65cfdb..eaa00c2c0 100644 --- a/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java +++ b/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java @@ -22,6 +22,7 @@ import io.cryostat.ConfigProperties; import io.cryostat.core.reports.InterruptibleReportGenerator.AnalysisResult; +import io.cryostat.diagnostic.DiagnosticsHelper; import io.cryostat.recordings.ArchivedRecordings.ArchivedRecording; import io.cryostat.reports.AnalysisReportAggregator; import io.cryostat.reports.ReportsService; @@ -59,17 +60,23 @@ public class LongRunningRequestGenerator { public static final String ARCHIVED_REPORT_COMPLETE_ADDRESS = "io.cryostat.recording.LongRunningRequestGenerator.ArchivedReportComplete"; + public static final String THREAD_DUMP_ADDRESS = + "io.cryostat.recordings.LongRunningRequestGenerator.ThreadDump"; + private static final String ARCHIVE_RECORDING_SUCCESS = "ArchiveRecordingSuccess"; private static final String ARCHIVE_RECORDING_FAIL = "ArchiveRecordingFailure"; private static final String GRAFANA_UPLOAD_SUCCESS = "GrafanaUploadSuccess"; private static final String GRAFANA_UPLOAD_FAIL = "GrafanaUploadFailure"; private static final String REPORT_SUCCESS = "ReportSuccess"; private static final String REPORT_FAILURE = "ReportFailure"; + private static final String THREAD_DUMP_SUCCESS = "ThreadDumpSuccess"; + private static final String THREAD_DUMP_FAILURE = "ThreadDumpFailure"; @Inject Logger logger; @Inject private EventBus bus; @Inject private RecordingHelper recordingHelper; @Inject private ReportsService reportsService; + @Inject private DiagnosticsHelper diagnosticsHelper; @Inject AnalysisReportAggregator analysisReportAggregator; @ConfigProperty(name = ConfigProperties.CONNECTIONS_FAILED_TIMEOUT) @@ -80,6 +87,34 @@ public class LongRunningRequestGenerator { public LongRunningRequestGenerator() {} + @ConsumeEvent(value = THREAD_DUMP_ADDRESS, blocking = true) + @Transactional + public void onMessage(ThreadDumpRequest request) { + logger.trace("Job ID: " + request.id() + " submitted."); + try { + var target = Target.findById(request.jvmId); + var dump = diagnosticsHelper.dumpThreads(request.format, target.id); + logger.trace("Thread Dump complete, firing notification"); + bus.publish( + MessagingServer.class.getName(), + new Notification( + THREAD_DUMP_SUCCESS, + Map.of( + "jobId", + request.id(), + "targetId", + dump.jvmId(), + "downloadUrl", + dump.downloadUrl()))); + } catch (Exception e) { + logger.warn("Failed to dump threads"); + bus.publish( + MessagingServer.class.getName(), + new Notification(THREAD_DUMP_FAILURE, Map.of("jobId", request.id()))); + throw new CompletionException(e); + } + } + @ConsumeEvent(value = ARCHIVE_REQUEST_ADDRESS, blocking = true) @Transactional public ArchivedRecording onMessage(ArchiveRequest request) { @@ -338,4 +373,12 @@ public record ArchivedReportCompletion( Objects.requireNonNull(report); } } + + public record ThreadDumpRequest(String id, String jvmId, String format) { + public ThreadDumpRequest { + Objects.requireNonNull(id); + Objects.requireNonNull(jvmId); + Objects.requireNonNull(format); + } + } } From 4fe54df58fbfe65df59e66e222b316bb2904bcc4 Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Mon, 2 Jun 2025 13:08:38 -0400 Subject: [PATCH 04/22] Fix error notification class --- src/main/java/io/cryostat/diagnostic/Diagnostics.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/io/cryostat/diagnostic/Diagnostics.java b/src/main/java/io/cryostat/diagnostic/Diagnostics.java index acfdcc5a3..17a76db98 100644 --- a/src/main/java/io/cryostat/diagnostic/Diagnostics.java +++ b/src/main/java/io/cryostat/diagnostic/Diagnostics.java @@ -98,10 +98,7 @@ public String threadDump( new ThreadDumpRequest( UUID.randomUUID().toString(), Long.toString(targetId), format); response.endHandler( - (e) -> - bus.publish( - LongRunningRequestGenerator.GRAFANA_ARCHIVE_REQUEST_ADDRESS, - request)); + (e) -> bus.publish(LongRunningRequestGenerator.THREAD_DUMP_ADDRESS, request)); return request.id(); } @@ -116,7 +113,7 @@ public List getThreadDumps(@RestPath long targetId) { @DELETE @Blocking - @Path("/{threadDumpId}") + @Path("/threaddump/{threadDumpId}") @RolesAllowed("write") public void deleteThreadDump(@RestPath String threadDumpId) { try { From fb193d53d86a9993bd4cb3893106c7c20f40b839 Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Mon, 2 Jun 2025 16:41:19 -0400 Subject: [PATCH 05/22] Fix downloads, filter by target, temp debug logging --- .../io/cryostat/diagnostic/Diagnostics.java | 32 +++++++++------ .../diagnostic/DiagnosticsHelper.java | 41 ++++++++++++------- .../LongRunningRequestGenerator.java | 1 - 3 files changed, 47 insertions(+), 27 deletions(-) diff --git a/src/main/java/io/cryostat/diagnostic/Diagnostics.java b/src/main/java/io/cryostat/diagnostic/Diagnostics.java index 17a76db98..9ab6703c5 100644 --- a/src/main/java/io/cryostat/diagnostic/Diagnostics.java +++ b/src/main/java/io/cryostat/diagnostic/Diagnostics.java @@ -62,7 +62,7 @@ import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; -@Path("/api/beta/diagnostics/targets/{targetId}") +@Path("/api/beta/diagnostics/") public class Diagnostics { @Inject TargetConnectionManager targetConnectionManager; @@ -87,13 +87,13 @@ public class Diagnostics { @Inject EventBus bus; @Inject DiagnosticsHelper helper; - @Path("/threaddump") + @Path("targets/{targetId}/threaddump") @RolesAllowed("write") @Blocking @POST public String threadDump( HttpServerResponse response, @RestPath long targetId, @RestQuery String format) { - log.trace("Creating new thread dump request"); + log.trace("Creating new thread dump request for target: " + targetId); ThreadDumpRequest request = new ThreadDumpRequest( UUID.randomUUID().toString(), Long.toString(targetId), format); @@ -102,21 +102,24 @@ public String threadDump( return request.id(); } - @Path("/threaddump") + @Path("targets/{targetId}/threaddump") @RolesAllowed("read") @Blocking @GET public List getThreadDumps(@RestPath long targetId) { - log.trace("Fetching thread dumps"); - return helper.getThreadDumps(bucket); + log.warn("Fetching thread dumps for target: " + targetId); + log.warn("Thread dumps: " + helper.getThreadDumps(targetId)); + log.warn("Storage bucket: " + bucket); + return helper.getThreadDumps(targetId); } @DELETE @Blocking - @Path("/threaddump/{threadDumpId}") + @Path("targets/{targetId}/threaddump/{threadDumpId}") @RolesAllowed("write") public void deleteThreadDump(@RestPath String threadDumpId) { try { + log.warn("Deleting thread dump with ID: " + threadDumpId); storage.headObject( HeadObjectRequest.builder().bucket(bucket).key(threadDumpId).build()); } catch (NoSuchKeyException e) { @@ -133,9 +136,13 @@ public void deleteThreadDump(@RestPath String threadDumpId) { public RestResponse handleStorageDownload( @RestPath String encodedKey, @RestQuery String query) throws URISyntaxException { Pair decodedKey = helper.decodedKey(encodedKey); - String key = helper.threadDumpKey(decodedKey); - - storage.headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build()) + var threadDumpId = decodedKey.getValue().strip(); + log.warn("Handling download Request for encodedKey: " + encodedKey); + log.warn("Handling download Request for query: " + query); + log.warn("Decoded key: " + decodedKey.toString()); + log.warn("UUID: " + threadDumpId); + log.warn("Bucket: " + bucket); + storage.headObject(HeadObjectRequest.builder().bucket(bucket).key(threadDumpId).build()) .sdkHttpResponse(); if (!presignedDownloadsEnabled) { @@ -149,7 +156,8 @@ public RestResponse handleStorageDownload( } log.tracev("Handling presigned download request for {0}", decodedKey); - GetObjectRequest getRequest = GetObjectRequest.builder().bucket(bucket).key(key).build(); + GetObjectRequest getRequest = + GetObjectRequest.builder().bucket(bucket).key(threadDumpId).build(); GetObjectPresignRequest presignRequest = GetObjectPresignRequest.builder() .signatureDuration(Duration.ofMinutes(1)) @@ -185,7 +193,7 @@ public RestResponse handleStorageDownload( return response.location(uri).build(); } - @Path("/gc") + @Path("targets/{targetId}/gc") @RolesAllowed("write") @Blocking @POST diff --git a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java index a5493757b..1994f4a10 100644 --- a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java +++ b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java @@ -40,7 +40,6 @@ import jakarta.ws.rs.BadRequestException; import jakarta.ws.rs.core.MediaType; import org.apache.commons.codec.binary.Base64; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.eclipse.microprofile.config.inject.ConfigProperty; import org.jboss.logging.Logger; @@ -78,6 +77,7 @@ public class DiagnosticsHelper { @Inject StorageBuckets buckets; void onStart(@Observes StartupEvent evt) { + log.warn("Creating thread dump bucket: " + bucket); buckets.createIfNecessary(bucket); } @@ -85,6 +85,8 @@ public ThreadDump dumpThreads(String format, long targetId) { if (!(format.equals(DUMP_THREADS) || format.equals(DUMP_THREADS_TO_FIlE))) { throw new BadRequestException(); } + log.warn( + "Thread Dump request received for Target: " + targetId + " with format: " + format); Object[] params = new Object[1]; String[] signature = new String[] {String[].class.getName()}; return targetConnectionManager.executeConnectedTask( @@ -97,8 +99,8 @@ public ThreadDump dumpThreads(String format, long targetId) { }); } - public List getThreadDumps(String jvmId) { - return listThreadDumps(jvmId).stream() + public List getThreadDumps(long targetId) { + return listThreadDumps(targetId).stream() .map( item -> { try { @@ -108,6 +110,13 @@ public List getThreadDumps(String jvmId) { return null; } }) + .filter( + item -> { + log.warn("Item jvmID: " + item.jvmId()); + log.warn("Item key: " + item.uuid()); + log.warn("Item download URL: " + item.downloadUrl()); + return Target.findById(targetId).jvmId.equals(item.jvmId()); + }) .toList(); } @@ -166,6 +175,9 @@ public ThreadDump addThreadDump(String content, String jvmId) { .tagging(createTagging(jvmId, uuid)) .build(), RequestBody.fromString(content)); + log.warn("Putting Thread dump into storage with key: " + uuid); + log.warn("jvmID: " + jvmId); + log.warn("Bucket: " + bucket); return new ThreadDump(content, jvmId, downloadUrl(jvmId, uuid), uuid); } @@ -195,18 +207,19 @@ private Tagging createTagging(String jvmId, String uuid) { } public String downloadUrl(String jvmId, String filename) { - return String.format("/threaddump/download/%s", encodedKey(jvmId, filename)); + return String.format( + "/api/beta/diagnostics/threaddump/download/%s", encodedKey(jvmId, filename)); } - public String encodedKey(String jvmId, String filename) { + public String encodedKey(String jvmId, String uuid) { Objects.requireNonNull(jvmId); - Objects.requireNonNull(filename); + Objects.requireNonNull(uuid); return base64Url.encodeAsString( - (threadDumpKey(jvmId, filename)).getBytes(StandardCharsets.UTF_8)); + (threadDumpKey(jvmId, uuid)).getBytes(StandardCharsets.UTF_8)); } - public String threadDumpKey(String jvmId, String filename) { - return (jvmId + "/" + filename).strip(); + public String threadDumpKey(String jvmId, String uuid) { + return (jvmId + "/" + uuid).strip(); } public String threadDumpKey(Pair pair) { @@ -218,7 +231,8 @@ public InputStream getThreadDumpStream(String jvmId, String threadDumpID) { } public InputStream getThreadDumpStream(String encodedKey) { - String key = new String(base64Url.decode(encodedKey), StandardCharsets.UTF_8); + Pair decodedKey = decodedKey(encodedKey); + var key = decodedKey.getValue().strip(); GetObjectRequest getRequest = GetObjectRequest.builder().bucket(bucket).key(key).build(); @@ -234,11 +248,10 @@ public Pair decodedKey(String encodedKey) { return Pair.of(parts[0], parts[1]); } - public List listThreadDumps(String jvmId) { + public List listThreadDumps(long targetId) { var builder = ListObjectsV2Request.builder().bucket(bucket); - if (StringUtils.isNotBlank(jvmId)) { - builder = builder.prefix(jvmId); - } + var jvmId = Target.findById(targetId).jvmId; + log.warn("Listing thread dumps for jvmId: " + jvmId); return storage.listObjectsV2(builder.build()).contents().stream().toList(); } } diff --git a/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java b/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java index eaa00c2c0..7e5937510 100644 --- a/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java +++ b/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java @@ -94,7 +94,6 @@ public void onMessage(ThreadDumpRequest request) { try { var target = Target.findById(request.jvmId); var dump = diagnosticsHelper.dumpThreads(request.format, target.id); - logger.trace("Thread Dump complete, firing notification"); bus.publish( MessagingServer.class.getName(), new Notification( From dd549aa57d1503ac81fdbe24ad49127059b8b4d0 Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Tue, 17 Jun 2025 10:19:44 -0400 Subject: [PATCH 06/22] Support Metadata storage mode, move logging to tracev --- .../java/io/cryostat/ConfigProperties.java | 7 + .../BucketedDiagnosticsMetadataService.java | 146 +++++++++++++++ .../io/cryostat/diagnostic/Diagnostics.java | 20 +- .../diagnostic/DiagnosticsHelper.java | 177 +++++++++++++----- .../java/io/cryostat/util/CRUDService.java | 44 +++++ 5 files changed, 337 insertions(+), 57 deletions(-) create mode 100644 src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java create mode 100644 src/main/java/io/cryostat/util/CRUDService.java diff --git a/src/main/java/io/cryostat/ConfigProperties.java b/src/main/java/io/cryostat/ConfigProperties.java index b800264fe..58d970d96 100644 --- a/src/main/java/io/cryostat/ConfigProperties.java +++ b/src/main/java/io/cryostat/ConfigProperties.java @@ -16,12 +16,16 @@ package io.cryostat; public class ConfigProperties { + public static final String STORAGE_METADATA_STORAGE_MODE = "storage.metadata.storage-mode"; public static final String AWS_BUCKET_NAME_ARCHIVES = "storage.buckets.archives.name"; + public static final String AWS_BUCKET_NAME_METADATA = "storage.buckets.metadata.name"; public static final String AWS_BUCKET_NAME_EVENT_TEMPLATES = "storage.buckets.event-templates.name"; public static final String AWS_BUCKET_NAME_PROBE_TEMPLATES = "storage.buckets.probe-templates.name"; public static final String AWS_BUCKET_NAME_THREAD_DUMPS = "storage.buckets.thread-dumps.name"; + public static final String AWS_METADATA_PREFIX_THREAD_DUMPS = + "storage.metadata.prefix.thread-dumps"; public static final String CONTAINERS_POLL_PERIOD = "cryostat.discovery.containers.poll-period"; public static final String CONTAINERS_REQUEST_TIMEOUT = @@ -53,6 +57,9 @@ public class ConfigProperties { public static final String STORAGE_PRESIGNED_DOWNLOADS_ENABLED = "storage.presigned-downloads.enabled"; + public static final String STORAGE_METADATA_THREAD_DUMPS_STORAGE_MODE = + "storage.metadata.thread-dumps.storage.mode"; + public static final String CUSTOM_TEMPLATES_DIR = "templates-dir"; public static final String PRESET_TEMPLATES_DIR = "preset-templates-dir"; public static final String PROBE_TEMPLATES_DIR = "probe-templates-dir"; diff --git a/src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java b/src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java new file mode 100644 index 000000000..c1bf0324b --- /dev/null +++ b/src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java @@ -0,0 +1,146 @@ +/* + * Copyright The Cryostat Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cryostat.diagnostic; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import io.cryostat.ConfigProperties; +import io.cryostat.StorageBuckets; +import io.cryostat.diagnostic.Diagnostics.ThreadDump; +import io.cryostat.util.CRUDService; +import io.cryostat.util.HttpMimeType; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.quarkus.arc.lookup.LookupIfProperty; +import io.quarkus.runtime.StartupEvent; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Observes; +import jakarta.inject.Inject; +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.jboss.logging.Logger; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; + +@ApplicationScoped +@LookupIfProperty( + name = ConfigProperties.STORAGE_METADATA_THREAD_DUMPS_STORAGE_MODE, + stringValue = DiagnosticsHelper.METADATA_STORAGE_MODE_BUCKET) +public class BucketedDiagnosticsMetadataService + implements CRUDService { + + @ConfigProperty(name = ConfigProperties.STORAGE_METADATA_THREAD_DUMPS_STORAGE_MODE) + String storageMode; + + @ConfigProperty(name = ConfigProperties.AWS_BUCKET_NAME_METADATA) + String bucket; + + @ConfigProperty(name = ConfigProperties.AWS_METADATA_PREFIX_THREAD_DUMPS) + String prefix; + + @Inject S3Client storage; + @Inject StorageBuckets buckets; + + @Inject ObjectMapper mapper; + + @Inject Logger logger; + + void onStart(@Observes StartupEvent evt) { + if (!DiagnosticsHelper.METADATA_STORAGE_MODE_BUCKET.equals(storageMode)) { + return; + } + buckets.createIfNecessary(bucket); + } + + @Override + public List list() throws IOException { + var builder = ListObjectsV2Request.builder().bucket(bucket).prefix(prefix); + var objs = storage.listObjectsV2(builder.build()).contents(); + return objs.stream() + .map( + t -> { + // TODO this entails a remote file read over the network and then some + // minor processing of the received file. More time will be spent + // retrieving the data than processing it, so this should be + // parallelized. + try { + return read(t.key()).orElseThrow(); + } catch (IOException e) { + logger.error(e); + return null; + } + }) + .filter(Objects::nonNull) + .toList(); + } + + @Override + public void create(String k, ThreadDump threadDump) throws IOException { + storage.putObject( + PutObjectRequest.builder() + .bucket(bucket) + .key(prefix(k)) + .contentType(HttpMimeType.JFC.mime()) + .build(), + RequestBody.fromBytes(mapper.writeValueAsBytes(ThreadDumpMeta.from(threadDump)))); + } + + @Override + public Optional read(String k) throws IOException { + try (var stream = + new BufferedInputStream( + storage.getObject( + GetObjectRequest.builder() + .bucket(bucket) + .key(prefix(k)) + .build()))) { + return Optional.of(mapper.readValue(stream, ThreadDumpMeta.class)) + .map(ThreadDumpMeta::asThreadDump); + } + } + + @Override + public void delete(String k) throws IOException { + storage.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(prefix(k)).build()); + } + + private String prefix(String key) { + return String.format("%s/%s", prefix, key); + } + + // just a thin serialization adapter. Jackson ObjectMapper complains about not being able to + // instantiate the Template type directly. + static record ThreadDumpMeta(String uuid, String jvmId, String content, String downloadUrl) { + static ThreadDumpMeta from(ThreadDump threadDump) { + return new ThreadDumpMeta( + threadDump.uuid(), + threadDump.jvmId(), + threadDump.content(), + threadDump.downloadUrl()); + } + + ThreadDump asThreadDump() { + return new ThreadDump(content, jvmId, downloadUrl, uuid); + } + } +} diff --git a/src/main/java/io/cryostat/diagnostic/Diagnostics.java b/src/main/java/io/cryostat/diagnostic/Diagnostics.java index 9ab6703c5..f8d6e5318 100644 --- a/src/main/java/io/cryostat/diagnostic/Diagnostics.java +++ b/src/main/java/io/cryostat/diagnostic/Diagnostics.java @@ -93,7 +93,7 @@ public class Diagnostics { @POST public String threadDump( HttpServerResponse response, @RestPath long targetId, @RestQuery String format) { - log.trace("Creating new thread dump request for target: " + targetId); + log.tracev("Creating new thread dump request for target: {0}", targetId); ThreadDumpRequest request = new ThreadDumpRequest( UUID.randomUUID().toString(), Long.toString(targetId), format); @@ -107,9 +107,9 @@ public String threadDump( @Blocking @GET public List getThreadDumps(@RestPath long targetId) { - log.warn("Fetching thread dumps for target: " + targetId); - log.warn("Thread dumps: " + helper.getThreadDumps(targetId)); - log.warn("Storage bucket: " + bucket); + log.tracev("Fetching thread dumps for target: {0}", targetId); + log.tracev("Thread dumps: {0}", helper.getThreadDumps(targetId)); + log.tracev("Storage bucket: {0}", bucket); return helper.getThreadDumps(targetId); } @@ -119,7 +119,7 @@ public List getThreadDumps(@RestPath long targetId) { @RolesAllowed("write") public void deleteThreadDump(@RestPath String threadDumpId) { try { - log.warn("Deleting thread dump with ID: " + threadDumpId); + log.tracev("Deleting thread dump with ID: {0}", threadDumpId); storage.headObject( HeadObjectRequest.builder().bucket(bucket).key(threadDumpId).build()); } catch (NoSuchKeyException e) { @@ -137,11 +137,11 @@ public RestResponse handleStorageDownload( @RestPath String encodedKey, @RestQuery String query) throws URISyntaxException { Pair decodedKey = helper.decodedKey(encodedKey); var threadDumpId = decodedKey.getValue().strip(); - log.warn("Handling download Request for encodedKey: " + encodedKey); - log.warn("Handling download Request for query: " + query); - log.warn("Decoded key: " + decodedKey.toString()); - log.warn("UUID: " + threadDumpId); - log.warn("Bucket: " + bucket); + log.tracev("Handling download Request for encodedKey: {0}", encodedKey); + log.tracev("Handling download Request for query: {0}", query); + log.tracev("Decoded key: {0}", decodedKey.toString()); + log.tracev("UUID: {0}", threadDumpId); + log.tracev("Bucket: {0}", bucket); storage.headObject(HeadObjectRequest.builder().bucket(bucket).key(threadDumpId).build()) .sdkHttpResponse(); diff --git a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java index 1994f4a10..266691c49 100644 --- a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java +++ b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java @@ -19,6 +19,7 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; @@ -35,6 +36,7 @@ import io.vertx.mutiny.core.eventbus.EventBus; import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.event.Observes; +import jakarta.enterprise.inject.Instance; import jakarta.inject.Inject; import jakarta.inject.Named; import jakarta.ws.rs.BadRequestException; @@ -47,6 +49,7 @@ import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectTaggingRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.S3Object; @@ -59,6 +62,9 @@ public class DiagnosticsHelper { @ConfigProperty(name = ConfigProperties.AWS_BUCKET_NAME_THREAD_DUMPS) String bucket; + @ConfigProperty(name = ConfigProperties.STORAGE_METADATA_STORAGE_MODE) + String storageMode; + @Inject @Named(Producers.BASE64_URL) Base64 base64Url; @@ -66,18 +72,26 @@ public class DiagnosticsHelper { @Inject S3Client storage; @Inject Logger log; + @Inject Instance metadataService; + private static final String DUMP_THREADS = "threadPrint"; private static final String DUMP_THREADS_TO_FIlE = "threadDumpToFile"; private static final String DIAGNOSTIC_BEAN_NAME = "com.sun.management:type=DiagnosticCommand"; static final String THREAD_DUMP_REQUESTED = "ThreadDumpRequested"; static final String THREAD_DUMP_DELETED = "ThreadDumpDeleted"; + private static final String META_KEY_NAME = "uuid"; + private static final String META_KEY_JVMID = "jvmId"; + + public static final String METADATA_STORAGE_MODE_TAGGING = "tagging"; + public static final String METADATA_STORAGE_MODE_OBJECTMETA = "metadata"; + public static final String METADATA_STORAGE_MODE_BUCKET = "bucket"; @Inject EventBus bus; @Inject TargetConnectionManager targetConnectionManager; @Inject StorageBuckets buckets; void onStart(@Observes StartupEvent evt) { - log.warn("Creating thread dump bucket: " + bucket); + log.tracev("Creating thread dump bucket: {0}", bucket); buckets.createIfNecessary(bucket); } @@ -85,8 +99,8 @@ public ThreadDump dumpThreads(String format, long targetId) { if (!(format.equals(DUMP_THREADS) || format.equals(DUMP_THREADS_TO_FIlE))) { throw new BadRequestException(); } - log.warn( - "Thread Dump request received for Target: " + targetId + " with format: " + format); + log.tracev( + "Thread Dump request received for Target: {0} with format: {1}", targetId, format); Object[] params = new Object[1]; String[] signature = new String[] {String[].class.getName()}; return targetConnectionManager.executeConnectedTask( @@ -112,46 +126,69 @@ public List getThreadDumps(long targetId) { }) .filter( item -> { - log.warn("Item jvmID: " + item.jvmId()); - log.warn("Item key: " + item.uuid()); - log.warn("Item download URL: " + item.downloadUrl()); + log.tracev("Item jvmID: {0}", item.jvmId()); + log.tracev("Item key: {0}", item.uuid()); + log.tracev("Item download URL: {0}", item.downloadUrl()); return Target.findById(targetId).jvmId.equals(item.jvmId()); }) + .filter(item -> !Objects.isNull(item)) .toList(); } private ThreadDump convertObject(S3Object object) throws Exception { - var req = GetObjectTaggingRequest.builder().bucket(bucket).key(object.key()).build(); - var tagging = storage.getObjectTagging(req); - var list = tagging.tagSet(); - if (!tagging.hasTagSet() || list.isEmpty()) { - throw new Exception("No metadata found"); + String jvmId, uuid; + switch (storageMode(storageMode)) { + case StorageMode.TAGGING: + var req = + GetObjectTaggingRequest.builder().bucket(bucket).key(object.key()).build(); + var tagging = storage.getObjectTagging(req); + var list = tagging.tagSet(); + if (!tagging.hasTagSet() || list.isEmpty()) { + throw new Exception("No metadata found"); + } + var decodedList = new ArrayList>(); + list.forEach( + t -> { + var encodedKey = t.key(); + var decodedKey = + new String(base64Url.decode(encodedKey), StandardCharsets.UTF_8) + .trim(); + var encodedValue = t.value(); + var decodedValue = + new String( + base64Url.decode(encodedValue), + StandardCharsets.UTF_8) + .trim(); + decodedList.add(Pair.of(decodedKey, decodedValue)); + }); + jvmId = + decodedList.stream() + .filter(t -> t.getKey().equals("jvmId")) + .map(Pair::getValue) + .findFirst() + .orElseThrow(); + uuid = + decodedList.stream() + .filter(t -> t.getKey().equals("uuid")) + .map(Pair::getValue) + .findFirst() + .orElseThrow(); + // content, jvmid, downloadurl, uuid + break; + case StorageMode.METADATA: + var headReq = HeadObjectRequest.builder().bucket(bucket).key(object.key()).build(); + var meta = storage.headObject(headReq).metadata(); + uuid = Objects.requireNonNull(meta.get(META_KEY_NAME)); + jvmId = Objects.requireNonNull(meta.get(META_KEY_JVMID)); + break; + case StorageMode.BUCKET: + var t = metadataService.get().read(object.key()).orElseThrow(); + uuid = t.uuid(); + jvmId = t.jvmId(); + break; + default: + throw new IllegalStateException(); } - var decodedList = new ArrayList>(); - list.forEach( - t -> { - var encodedKey = t.key(); - var decodedKey = - new String(base64Url.decode(encodedKey), StandardCharsets.UTF_8).trim(); - var encodedValue = t.value(); - var decodedValue = - new String(base64Url.decode(encodedValue), StandardCharsets.UTF_8) - .trim(); - decodedList.add(Pair.of(decodedKey, decodedValue)); - }); - var jvmId = - decodedList.stream() - .filter(t -> t.getKey().equals("jvmId")) - .map(Pair::getValue) - .findFirst() - .orElseThrow(); - var uuid = - decodedList.stream() - .filter(t -> t.getKey().equals("uuid")) - .map(Pair::getValue) - .findFirst() - .orElseThrow(); - // content, jvmid, downloadurl, uuid return new ThreadDump(getThreadDumpContent(uuid), jvmId, downloadUrl(jvmId, uuid), uuid); } @@ -167,17 +204,37 @@ private InputStream getModel(String name) { public ThreadDump addThreadDump(String content, String jvmId) { String uuid = UUID.randomUUID().toString(); - storage.putObject( + var reqBuilder = PutObjectRequest.builder() .bucket(bucket) .key(uuid) - .contentType(MediaType.TEXT_PLAIN) - .tagging(createTagging(jvmId, uuid)) - .build(), - RequestBody.fromString(content)); - log.warn("Putting Thread dump into storage with key: " + uuid); - log.warn("jvmID: " + jvmId); - log.warn("Bucket: " + bucket); + .contentType(MediaType.TEXT_PLAIN); + switch (storageMode(storageMode)) { + case StorageMode.TAGGING: + reqBuilder = reqBuilder.tagging(createTagging(jvmId, uuid)); + log.tracev("Putting Thread dump into storage with key: {0}", uuid); + log.tracev("jvmID: {0}", jvmId); + log.tracev("Bucket: {0}", bucket); + break; + case StorageMode.METADATA: + reqBuilder = + reqBuilder.metadata(Map.of(META_KEY_NAME, uuid, META_KEY_JVMID, jvmId)); + break; + case StorageMode.BUCKET: + try { + metadataService + .get() + .create( + uuid, + new ThreadDump(content, jvmId, downloadUrl(jvmId, uuid), uuid)); + break; + } catch (IOException ioe) { + log.warnv("Exception thrown while adding thread dump to storage: {0}", ioe); + } + default: + throw new IllegalStateException(); + } + storage.putObject(reqBuilder.build(), RequestBody.fromString(content)); return new ThreadDump(content, jvmId, downloadUrl(jvmId, uuid), uuid); } @@ -249,9 +306,35 @@ public Pair decodedKey(String encodedKey) { } public List listThreadDumps(long targetId) { - var builder = ListObjectsV2Request.builder().bucket(bucket); var jvmId = Target.findById(targetId).jvmId; - log.warn("Listing thread dumps for jvmId: " + jvmId); - return storage.listObjectsV2(builder.build()).contents().stream().toList(); + log.tracev("Listing thread dumps for jvmId: {0}", jvmId); + return storage + .listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).build()) + .contents() + .stream() + .toList(); + } + + public static StorageMode storageMode(String name) { + return Arrays.asList(StorageMode.values()).stream() + .filter(s -> s.name().equalsIgnoreCase(name)) + .findFirst() + .orElseThrow(); + } + + static enum StorageMode { + TAGGING(METADATA_STORAGE_MODE_TAGGING), + METADATA(METADATA_STORAGE_MODE_OBJECTMETA), + BUCKET(METADATA_STORAGE_MODE_BUCKET), + ; + private final String key; + + private StorageMode(String key) { + this.key = key; + } + + public String getKey() { + return key; + } } } diff --git a/src/main/java/io/cryostat/util/CRUDService.java b/src/main/java/io/cryostat/util/CRUDService.java new file mode 100644 index 000000000..0ff735ee8 --- /dev/null +++ b/src/main/java/io/cryostat/util/CRUDService.java @@ -0,0 +1,44 @@ +/* + * Copyright The Cryostat Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cryostat.util; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Stream; + +public interface CRUDService { + + default List list() throws IOException { + return List.of(); + } + + default Stream find(Predicate p) throws IOException { + return list().stream().filter(p); + } + + void create(Key k, InValue v) throws IOException; + + Optional read(Key k) throws IOException; + + default void update(Key k, InValue v) throws IOException { + delete(k); + create(k, v); + } + + void delete(Key k) throws IOException; +} From 9233076951f4f45b112cedc1ecd2f0b77af6fac4 Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Tue, 17 Jun 2025 10:29:39 -0400 Subject: [PATCH 07/22] close inputStream --- src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java index 266691c49..9929f81ca 100644 --- a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java +++ b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java @@ -193,8 +193,9 @@ private ThreadDump convertObject(S3Object object) throws Exception { } public String getThreadDumpContent(String uuid) throws IOException { - InputStream is = getModel(uuid); - return new String(is.readAllBytes(), StandardCharsets.UTF_8); + try (InputStream is = getModel(uuid)) { + return new String(is.readAllBytes(), StandardCharsets.UTF_8); + } } private InputStream getModel(String name) { From f58cf3c69f7107d8469443943a3f7cbb9ebeb9c0 Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Wed, 18 Jun 2025 15:32:59 -0400 Subject: [PATCH 08/22] Remove embedded thread dump content, fix smoketest --- compose/cryostat.yml | 1 + compose/s3-ext.yml | 22 ++++++++++++ compose/s3-gw.yml | 34 +++++++++++++++++++ .../java/io/cryostat/ConfigProperties.java | 2 +- .../BucketedDiagnosticsMetadataService.java | 8 ++--- .../io/cryostat/diagnostic/Diagnostics.java | 5 ++- .../diagnostic/DiagnosticsHelper.java | 15 +++++--- src/main/resources/application.properties | 9 +++++ 8 files changed, 84 insertions(+), 12 deletions(-) create mode 100644 compose/s3-ext.yml create mode 100644 compose/s3-gw.yml diff --git a/compose/cryostat.yml b/compose/cryostat.yml index 918870378..af5364062 100644 --- a/compose/cryostat.yml +++ b/compose/cryostat.yml @@ -23,6 +23,7 @@ services: QUARKUS_HTTP_HOST: "cryostat" QUARKUS_HTTP_PORT: ${CRYOSTAT_HTTP_PORT} QUARKUS_HIBERNATE_ORM_LOG_SQL: "true" + STORAGE_METADATA_STORAGE_MODE: ${CRYOSTAT_STORAGE_MODE:-tagging} CRYOSTAT_DISCOVERY_JDP_ENABLED: ${CRYOSTAT_DISCOVERY_JDP_ENABLED:-true} CRYOSTAT_DISCOVERY_PODMAN_ENABLED: ${CRYOSTAT_DISCOVERY_PODMAN_ENABLED:-true} CRYOSTAT_DISCOVERY_DOCKER_ENABLED: ${CRYOSTAT_DISCOVERY_DOCKER_ENABLED:-true} diff --git a/compose/s3-ext.yml b/compose/s3-ext.yml new file mode 100644 index 000000000..87618cee3 --- /dev/null +++ b/compose/s3-ext.yml @@ -0,0 +1,22 @@ +services: + cryostat: + environment: + # the endpoint and credentials values below MUST be set by external environment variable + # before running the smoketest.bash script. The region should be overridden to match the + # endpoint, if necessary. The storage mode should be changed if the selected object storage + # provider does not support the object Tag API. + CRYOSTAT_ARCHIVED_RECORDINGS_METADATA_STORAGE_MODE: ${METADATA_STORAGE_MODE:-tagging} + QUARKUS_S3_ENDPOINT_OVERRIDE: ${S3_ENDPOINT} + # STORAGE_EXT_URL: /storage/ + QUARKUS_S3_PATH_STYLE_ACCESS: "false" + QUARKUS_S3_AWS_REGION: ${S3_REGION:-us-east-1} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + + # this stub is just here to satisfy the smoketest script. It is not actually involved in anything. + s3: + image: registry.access.redhat.com/ubi9/ubi-micro + hostname: s3 + command: /usr/bin/bash -c "while true; do sleep 5; done" + expose: + - "9000" \ No newline at end of file diff --git a/compose/s3-gw.yml b/compose/s3-gw.yml new file mode 100644 index 000000000..89df70e11 --- /dev/null +++ b/compose/s3-gw.yml @@ -0,0 +1,34 @@ +services: + cryostat: + depends_on: + s3: + condition: service_healthy + environment: + QUARKUS_S3_ENDPOINT_OVERRIDE: http://s3:7480 + STORAGE_EXT_URL: /storage/ + QUARKUS_S3_PATH_STYLE_ACCESS: "true" # needed since compose setup does not support DNS subdomain resolution + QUARKUS_S3_AWS_REGION: us-east-1 + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + s3: + image: ${S3GW_IMAGE:-quay.io/s3gw/s3gw:latest} + hostname: s3 + expose: + - "7480" + volumes: + - s3gw_data:/data + ulimits: + nofile: + soft: 4096 + hard: 4096 + restart: always + healthcheck: + test: curl --fail http://localhost:7480 || exit 1 + interval: 10s + retries: 3 + start_period: 30s + timeout: 5s + +volumes: + s3gw_data: + driver: local \ No newline at end of file diff --git a/src/main/java/io/cryostat/ConfigProperties.java b/src/main/java/io/cryostat/ConfigProperties.java index 694b3eb56..b81f3eef8 100644 --- a/src/main/java/io/cryostat/ConfigProperties.java +++ b/src/main/java/io/cryostat/ConfigProperties.java @@ -59,7 +59,7 @@ public class ConfigProperties { "storage.presigned-downloads.enabled"; public static final String STORAGE_METADATA_THREAD_DUMPS_STORAGE_MODE = - "storage.metadata.thread-dumps.storage.mode"; + "storage.metadata.thread-dumps.storage-mode"; public static final String CUSTOM_TEMPLATES_DIR = "templates-dir"; public static final String PRESET_TEMPLATES_DIR = "preset-templates-dir"; diff --git a/src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java b/src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java index c1bf0324b..cb3c70f71 100644 --- a/src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java +++ b/src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java @@ -130,17 +130,17 @@ private String prefix(String key) { // just a thin serialization adapter. Jackson ObjectMapper complains about not being able to // instantiate the Template type directly. - static record ThreadDumpMeta(String uuid, String jvmId, String content, String downloadUrl) { + static record ThreadDumpMeta(String uuid, String jvmId, String downloadUrl, long lastModified) { static ThreadDumpMeta from(ThreadDump threadDump) { return new ThreadDumpMeta( threadDump.uuid(), threadDump.jvmId(), - threadDump.content(), - threadDump.downloadUrl()); + threadDump.downloadUrl(), + threadDump.lastModified()); } ThreadDump asThreadDump() { - return new ThreadDump(content, jvmId, downloadUrl, uuid); + return new ThreadDump(jvmId, downloadUrl, uuid, lastModified); } } } diff --git a/src/main/java/io/cryostat/diagnostic/Diagnostics.java b/src/main/java/io/cryostat/diagnostic/Diagnostics.java index a53b4746d..1758d1bca 100644 --- a/src/main/java/io/cryostat/diagnostic/Diagnostics.java +++ b/src/main/java/io/cryostat/diagnostic/Diagnostics.java @@ -48,8 +48,8 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.eclipse.microprofile.config.inject.ConfigProperty; -import org.jboss.logging.Logger; import org.eclipse.microprofile.openapi.annotations.Operation; +import org.jboss.logging.Logger; import org.jboss.resteasy.reactive.RestPath; import org.jboss.resteasy.reactive.RestQuery; import org.jboss.resteasy.reactive.RestResponse; @@ -213,10 +213,9 @@ public void gc(@RestPath long targetId) { "java.lang:type=Memory", "gc", null, null, Void.class)); } - public record ThreadDump(String content, String jvmId, String downloadUrl, String uuid) { + public record ThreadDump(String jvmId, String downloadUrl, String uuid, long lastModified) { public ThreadDump { - Objects.requireNonNull(content); Objects.requireNonNull(jvmId); Objects.requireNonNull(downloadUrl); Objects.requireNonNull(uuid); diff --git a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java index 9929f81ca..a0823796d 100644 --- a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java +++ b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java @@ -29,6 +29,7 @@ import io.cryostat.Producers; import io.cryostat.StorageBuckets; import io.cryostat.diagnostic.Diagnostics.ThreadDump; +import io.cryostat.libcryostat.sys.Clock; import io.cryostat.targets.Target; import io.cryostat.targets.TargetConnectionManager; @@ -71,6 +72,7 @@ public class DiagnosticsHelper { @Inject S3Client storage; @Inject Logger log; + @Inject Clock clock; @Inject Instance metadataService; @@ -189,13 +191,14 @@ private ThreadDump convertObject(S3Object object) throws Exception { default: throw new IllegalStateException(); } - return new ThreadDump(getThreadDumpContent(uuid), jvmId, downloadUrl(jvmId, uuid), uuid); + return new ThreadDump( + jvmId, downloadUrl(jvmId, uuid), uuid, object.lastModified().toEpochMilli()); } public String getThreadDumpContent(String uuid) throws IOException { try (InputStream is = getModel(uuid)) { return new String(is.readAllBytes(), StandardCharsets.UTF_8); - } + } } private InputStream getModel(String name) { @@ -227,7 +230,11 @@ public ThreadDump addThreadDump(String content, String jvmId) { .get() .create( uuid, - new ThreadDump(content, jvmId, downloadUrl(jvmId, uuid), uuid)); + new ThreadDump( + jvmId, + downloadUrl(jvmId, uuid), + uuid, + clock.now().getEpochSecond())); break; } catch (IOException ioe) { log.warnv("Exception thrown while adding thread dump to storage: {0}", ioe); @@ -236,7 +243,7 @@ public ThreadDump addThreadDump(String content, String jvmId) { throw new IllegalStateException(); } storage.putObject(reqBuilder.build(), RequestBody.fromString(content)); - return new ThreadDump(content, jvmId, downloadUrl(jvmId, uuid), uuid); + return new ThreadDump(jvmId, downloadUrl(jvmId, uuid), uuid, clock.now().getEpochSecond()); } private Tagging createTagging(String jvmId, String uuid) { diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 7f144e44a..f2108d1a7 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -112,11 +112,20 @@ quarkus.http.filter.static.order=1 quarkus.s3.sync-client.type=apache storage-ext.url= storage.presigned-downloads.enabled=false +storage.metadata.storage-mode=tagging +storage.metadata.archives.storage-mode=${storage.metadata.storage-mode} +storage.metadata.event-templates.storage-mode=${storage.metadata.storage-mode} +storage.metadata.thread-dumps.storage-mode=${storage.metadata.storage-mode} +storage.metadata.thread-dumps.name=threaddumps storage.buckets.creation-retry.period=10s storage.buckets.archives.name=archivedrecordings storage.buckets.event-templates.name=eventtemplates storage.buckets.probe-templates.name=probes storage.buckets.thread-dumps.name=threaddumps +storage.buckets.metadata.name=metadata +storage.metadata.prefix.recordings=${storage.buckets.archives.name} +storage.metadata.prefix.event-templates=${storage.buckets.event-templates.name} +storage.metadata.prefix.thread-dumps=${storage.buckets.thread-dumps.name} quarkus.quinoa.build-dir=dist quarkus.quinoa.enable-spa-routing=true From 820d6dd064a6e8d4bb7241abf4f4515efb7b68bc Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Wed, 18 Jun 2025 16:13:06 -0400 Subject: [PATCH 09/22] Address review feedback --- .../BucketedDiagnosticsMetadataService.java | 23 +++---------------- .../io/cryostat/diagnostic/Diagnostics.java | 14 +++++------ .../diagnostic/DiagnosticsHelper.java | 6 ++--- .../LongRunningRequestGenerator.java | 6 ++--- .../io/cryostat/targets/AgentConnection.java | 1 + 5 files changed, 17 insertions(+), 33 deletions(-) diff --git a/src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java b/src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java index cb3c70f71..2ea3be986 100644 --- a/src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java +++ b/src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java @@ -100,9 +100,9 @@ public void create(String k, ThreadDump threadDump) throws IOException { PutObjectRequest.builder() .bucket(bucket) .key(prefix(k)) - .contentType(HttpMimeType.JFC.mime()) + .contentType(HttpMimeType.PLAINTEXT.mime()) .build(), - RequestBody.fromBytes(mapper.writeValueAsBytes(ThreadDumpMeta.from(threadDump)))); + RequestBody.fromBytes(mapper.writeValueAsBytes(threadDump))); } @Override @@ -114,8 +114,7 @@ public Optional read(String k) throws IOException { .bucket(bucket) .key(prefix(k)) .build()))) { - return Optional.of(mapper.readValue(stream, ThreadDumpMeta.class)) - .map(ThreadDumpMeta::asThreadDump); + return Optional.of(mapper.readValue(stream, ThreadDump.class)); } } @@ -127,20 +126,4 @@ public void delete(String k) throws IOException { private String prefix(String key) { return String.format("%s/%s", prefix, key); } - - // just a thin serialization adapter. Jackson ObjectMapper complains about not being able to - // instantiate the Template type directly. - static record ThreadDumpMeta(String uuid, String jvmId, String downloadUrl, long lastModified) { - static ThreadDumpMeta from(ThreadDump threadDump) { - return new ThreadDumpMeta( - threadDump.uuid(), - threadDump.jvmId(), - threadDump.downloadUrl(), - threadDump.lastModified()); - } - - ThreadDump asThreadDump() { - return new ThreadDump(jvmId, downloadUrl, uuid, lastModified); - } - } } diff --git a/src/main/java/io/cryostat/diagnostic/Diagnostics.java b/src/main/java/io/cryostat/diagnostic/Diagnostics.java index 1758d1bca..4fa573bbf 100644 --- a/src/main/java/io/cryostat/diagnostic/Diagnostics.java +++ b/src/main/java/io/cryostat/diagnostic/Diagnostics.java @@ -90,14 +90,12 @@ public class Diagnostics { @Path("targets/{targetId}/threaddump") @RolesAllowed("write") - @Blocking @POST public String threadDump( HttpServerResponse response, @RestPath long targetId, @RestQuery String format) { log.tracev("Creating new thread dump request for target: {0}", targetId); ThreadDumpRequest request = - new ThreadDumpRequest( - UUID.randomUUID().toString(), Long.toString(targetId), format); + new ThreadDumpRequest(UUID.randomUUID().toString(), targetId, format); response.endHandler( (e) -> bus.publish(LongRunningRequestGenerator.THREAD_DUMP_ADDRESS, request)); return request.id(); @@ -109,8 +107,6 @@ public String threadDump( @GET public List getThreadDumps(@RestPath long targetId) { log.tracev("Fetching thread dumps for target: {0}", targetId); - log.tracev("Thread dumps: {0}", helper.getThreadDumps(targetId)); - log.tracev("Storage bucket: {0}", bucket); return helper.getThreadDumps(targetId); } @@ -143,8 +139,12 @@ public RestResponse handleStorageDownload( log.tracev("Decoded key: {0}", decodedKey.toString()); log.tracev("UUID: {0}", threadDumpId); log.tracev("Bucket: {0}", bucket); - storage.headObject(HeadObjectRequest.builder().bucket(bucket).key(threadDumpId).build()) - .sdkHttpResponse(); + try { + storage.headObject(HeadObjectRequest.builder().bucket(bucket).key(threadDumpId).build()) + .sdkHttpResponse(); + } catch (NoSuchKeyException e) { + throw new NotFoundException(e); + } if (!presignedDownloadsEnabled) { return ResponseBuilder.ok() diff --git a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java index a0823796d..ed9410676 100644 --- a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java +++ b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java @@ -131,9 +131,9 @@ public List getThreadDumps(long targetId) { log.tracev("Item jvmID: {0}", item.jvmId()); log.tracev("Item key: {0}", item.uuid()); log.tracev("Item download URL: {0}", item.downloadUrl()); - return Target.findById(targetId).jvmId.equals(item.jvmId()); + return Target.getTargetById(targetId).jvmId.equals(item.jvmId()); }) - .filter(item -> !Objects.isNull(item)) + .filter(Objects::nonNull) .toList(); } @@ -314,7 +314,7 @@ public Pair decodedKey(String encodedKey) { } public List listThreadDumps(long targetId) { - var jvmId = Target.findById(targetId).jvmId; + var jvmId = Target.getTargetById(targetId).jvmId; log.tracev("Listing thread dumps for jvmId: {0}", jvmId); return storage .listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).build()) diff --git a/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java b/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java index 0127deb29..6047c2331 100644 --- a/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java +++ b/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java @@ -99,7 +99,7 @@ public LongRunningRequestGenerator() {} public void onMessage(ThreadDumpRequest request) { logger.trace("Job ID: " + request.id() + " submitted."); try { - var target = Target.findById(request.jvmId); + var target = Target.getTargetById(request.targetId); var dump = diagnosticsHelper.dumpThreads(request.format, target.id); bus.publish( MessagingServer.class.getName(), @@ -380,10 +380,10 @@ public record ArchivedReportCompletion( } } - public record ThreadDumpRequest(String id, String jvmId, String format) { + public record ThreadDumpRequest(String id, long targetId, String format) { public ThreadDumpRequest { Objects.requireNonNull(id); - Objects.requireNonNull(jvmId); + Objects.requireNonNull(targetId); Objects.requireNonNull(format); } } diff --git a/src/main/java/io/cryostat/targets/AgentConnection.java b/src/main/java/io/cryostat/targets/AgentConnection.java index 1a3003704..243a4cdbd 100644 --- a/src/main/java/io/cryostat/targets/AgentConnection.java +++ b/src/main/java/io/cryostat/targets/AgentConnection.java @@ -117,6 +117,7 @@ public JvmIdentifier getJvmIdentifier() throws IDException, IOException { } } + @Override public T invokeMBeanOperation( String beanName, String operation, From 3a9226ae495aaaf66c108644cf3fab40f74a341c Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Wed, 18 Jun 2025 16:24:22 -0400 Subject: [PATCH 10/22] Remove uuid from tagging since it's used as the key --- .../diagnostic/DiagnosticsHelper.java | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java index ed9410676..58e28f8f4 100644 --- a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java +++ b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java @@ -126,6 +126,7 @@ public List getThreadDumps(long targetId) { return null; } }) + .filter(Objects::nonNull) .filter( item -> { log.tracev("Item jvmID: {0}", item.jvmId()); @@ -133,12 +134,11 @@ public List getThreadDumps(long targetId) { log.tracev("Item download URL: {0}", item.downloadUrl()); return Target.getTargetById(targetId).jvmId.equals(item.jvmId()); }) - .filter(Objects::nonNull) .toList(); } private ThreadDump convertObject(S3Object object) throws Exception { - String jvmId, uuid; + String jvmId; switch (storageMode(storageMode)) { case StorageMode.TAGGING: var req = @@ -169,30 +169,25 @@ private ThreadDump convertObject(S3Object object) throws Exception { .map(Pair::getValue) .findFirst() .orElseThrow(); - uuid = - decodedList.stream() - .filter(t -> t.getKey().equals("uuid")) - .map(Pair::getValue) - .findFirst() - .orElseThrow(); // content, jvmid, downloadurl, uuid break; case StorageMode.METADATA: var headReq = HeadObjectRequest.builder().bucket(bucket).key(object.key()).build(); var meta = storage.headObject(headReq).metadata(); - uuid = Objects.requireNonNull(meta.get(META_KEY_NAME)); jvmId = Objects.requireNonNull(meta.get(META_KEY_JVMID)); break; case StorageMode.BUCKET: var t = metadataService.get().read(object.key()).orElseThrow(); - uuid = t.uuid(); jvmId = t.jvmId(); break; default: throw new IllegalStateException(); } return new ThreadDump( - jvmId, downloadUrl(jvmId, uuid), uuid, object.lastModified().toEpochMilli()); + jvmId, + downloadUrl(jvmId, object.key()), + object.key(), + object.lastModified().toEpochMilli()); } public String getThreadDumpContent(String uuid) throws IOException { @@ -215,7 +210,7 @@ public ThreadDump addThreadDump(String content, String jvmId) { .contentType(MediaType.TEXT_PLAIN); switch (storageMode(storageMode)) { case StorageMode.TAGGING: - reqBuilder = reqBuilder.tagging(createTagging(jvmId, uuid)); + reqBuilder = reqBuilder.tagging(createTagging(jvmId)); log.tracev("Putting Thread dump into storage with key: {0}", uuid); log.tracev("jvmID: {0}", jvmId); log.tracev("Bucket: {0}", bucket); @@ -246,8 +241,8 @@ public ThreadDump addThreadDump(String content, String jvmId) { return new ThreadDump(jvmId, downloadUrl(jvmId, uuid), uuid, clock.now().getEpochSecond()); } - private Tagging createTagging(String jvmId, String uuid) { - var map = Map.of("jvmId", jvmId, "uuid", uuid); + private Tagging createTagging(String jvmId) { + var map = Map.of("jvmId", jvmId); var tags = new ArrayList(); tags.addAll( map.entrySet().stream() From be2de39dd59e57b7596f4a66dca82075fae6145d Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Thu, 19 Jun 2025 13:07:18 -0400 Subject: [PATCH 11/22] Fix potential NPE in getThreadDumps --- .../java/io/cryostat/diagnostic/DiagnosticsHelper.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java index 58e28f8f4..3dd134765 100644 --- a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java +++ b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java @@ -116,7 +116,7 @@ public ThreadDump dumpThreads(String format, long targetId) { } public List getThreadDumps(long targetId) { - return listThreadDumps(targetId).stream() + return listThreadDumps().stream() .map( item -> { try { @@ -132,7 +132,7 @@ public List getThreadDumps(long targetId) { log.tracev("Item jvmID: {0}", item.jvmId()); log.tracev("Item key: {0}", item.uuid()); log.tracev("Item download URL: {0}", item.downloadUrl()); - return Target.getTargetById(targetId).jvmId.equals(item.jvmId()); + return Objects.equals(Target.getTargetById(targetId).jvmId, item.jvmId()); }) .toList(); } @@ -308,9 +308,7 @@ public Pair decodedKey(String encodedKey) { return Pair.of(parts[0], parts[1]); } - public List listThreadDumps(long targetId) { - var jvmId = Target.getTargetById(targetId).jvmId; - log.tracev("Listing thread dumps for jvmId: {0}", jvmId); + public List listThreadDumps() { return storage .listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).build()) .contents() From c3f660ab050cd5b18a295d44bc1d35e5d101b5da Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Thu, 19 Jun 2025 13:12:35 -0400 Subject: [PATCH 12/22] formatting --- src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java index 3dd134765..bfe1932ef 100644 --- a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java +++ b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java @@ -132,7 +132,8 @@ public List getThreadDumps(long targetId) { log.tracev("Item jvmID: {0}", item.jvmId()); log.tracev("Item key: {0}", item.uuid()); log.tracev("Item download URL: {0}", item.downloadUrl()); - return Objects.equals(Target.getTargetById(targetId).jvmId, item.jvmId()); + return Objects.equals( + Target.getTargetById(targetId).jvmId, item.jvmId()); }) .toList(); } From eb4220c29f2c14aee2c35c3bd747da5bdc80d8be Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Wed, 25 Jun 2025 14:02:08 -0400 Subject: [PATCH 13/22] refactor --- .../BucketedDiagnosticsMetadataService.java | 5 +++-- .../diagnostic/DiagnosticsHelper.java | 21 +------------------ 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java b/src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java index 2ea3be986..857a984cc 100644 --- a/src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java +++ b/src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java @@ -24,6 +24,7 @@ import io.cryostat.ConfigProperties; import io.cryostat.StorageBuckets; import io.cryostat.diagnostic.Diagnostics.ThreadDump; +import io.cryostat.recordings.ArchivedRecordingMetadataService; import io.cryostat.util.CRUDService; import io.cryostat.util.HttpMimeType; @@ -45,7 +46,7 @@ @ApplicationScoped @LookupIfProperty( name = ConfigProperties.STORAGE_METADATA_THREAD_DUMPS_STORAGE_MODE, - stringValue = DiagnosticsHelper.METADATA_STORAGE_MODE_BUCKET) + stringValue = ArchivedRecordingMetadataService.METADATA_STORAGE_MODE_BUCKET) public class BucketedDiagnosticsMetadataService implements CRUDService { @@ -66,7 +67,7 @@ public class BucketedDiagnosticsMetadataService @Inject Logger logger; void onStart(@Observes StartupEvent evt) { - if (!DiagnosticsHelper.METADATA_STORAGE_MODE_BUCKET.equals(storageMode)) { + if (!ArchivedRecordingMetadataService.METADATA_STORAGE_MODE_BUCKET.equals(storageMode)) { return; } buckets.createIfNecessary(bucket); diff --git a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java index 6f5cd8f53..ea6efbc86 100644 --- a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java +++ b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java @@ -30,6 +30,7 @@ import io.cryostat.StorageBuckets; import io.cryostat.diagnostic.Diagnostics.ThreadDump; import io.cryostat.libcryostat.sys.Clock; +import io.cryostat.recordings.ArchivedRecordingMetadataService.StorageMode; import io.cryostat.targets.Target; import io.cryostat.targets.TargetConnectionManager; @@ -84,10 +85,6 @@ public class DiagnosticsHelper { private static final String META_KEY_NAME = "uuid"; private static final String META_KEY_JVMID = "jvmId"; - public static final String METADATA_STORAGE_MODE_TAGGING = "tagging"; - public static final String METADATA_STORAGE_MODE_OBJECTMETA = "metadata"; - public static final String METADATA_STORAGE_MODE_BUCKET = "bucket"; - @Inject EventBus bus; @Inject TargetConnectionManager targetConnectionManager; @Inject StorageBuckets buckets; @@ -323,20 +320,4 @@ public static StorageMode storageMode(String name) { .findFirst() .orElseThrow(); } - - static enum StorageMode { - TAGGING(METADATA_STORAGE_MODE_TAGGING), - METADATA(METADATA_STORAGE_MODE_OBJECTMETA), - BUCKET(METADATA_STORAGE_MODE_BUCKET), - ; - private final String key; - - private StorageMode(String key) { - this.key = key; - } - - public String getKey() { - return key; - } - } } From e2bb3824a6ee9d3f1fa3e8a2c278a651dfdf2728 Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Fri, 4 Jul 2025 19:18:12 -0400 Subject: [PATCH 14/22] Thread Dump endpoint tests --- .../cryostat/diagnostics/ThreadDumpsTest.java | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 src/test/java/io/cryostat/diagnostics/ThreadDumpsTest.java diff --git a/src/test/java/io/cryostat/diagnostics/ThreadDumpsTest.java b/src/test/java/io/cryostat/diagnostics/ThreadDumpsTest.java new file mode 100644 index 000000000..39db5da8e --- /dev/null +++ b/src/test/java/io/cryostat/diagnostics/ThreadDumpsTest.java @@ -0,0 +1,136 @@ +/* + * Copyright The Cryostat Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cryostat.diagnostics; + +import static io.restassured.RestAssured.given; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeoutException; + +import io.cryostat.AbstractTransactionalTestBase; +import io.cryostat.diagnostic.Diagnostics; + +import io.quarkus.test.common.http.TestHTTPEndpoint; +import io.quarkus.test.junit.QuarkusTest; +import io.restassured.http.ContentType; +import jakarta.websocket.DeploymentException; +import org.hamcrest.Matchers; +import org.junit.Test; + +@QuarkusTest +@TestHTTPEndpoint(Diagnostics.class) +public class ThreadDumpsTest extends AbstractTransactionalTestBase { + + @Test + public void testListNone() { + int id = defineSelfCustomTarget(); + given().log() + .all() + .when() + .pathParams(Map.of("targetId", id)) + .get("targets/{targetId}/threaddump") + .then() + .log() + .all() + .and() + .assertThat() + .contentType(ContentType.JSON) + .statusCode(200) + .body("size()", Matchers.equalTo(0)); + } + + @Test + public void testCreate() + throws InterruptedException, IOException, DeploymentException, TimeoutException { + int id = defineSelfCustomTarget(); + String jobId = + given().log() + .all() + .when() + .pathParams(Map.of("targetId", id)) + .post("targets/{targetId}/threaddump") + .then() + .log() + .all() + .and() + .assertThat() + .contentType(ContentType.JSON) + .statusCode(200) + .body("size()", Matchers.equalTo(0)) + .extract() + .body() + .asString(); + + expectWebSocketNotification( + "ThreadDumpSuccess", + o -> jobId.equals(o.getJsonObject("message").getString("jobId"))); + } + + @Test + public void testListInvalid() { + given().log() + .all() + .when() + .pathParams(Map.of("targetId", -1)) + .get("targets/{targetId}/threaddump") + .then() + .log() + .all() + .and() + .assertThat() + .contentType(ContentType.JSON) + .statusCode(400); + } + + @Test + void testDeleteInvalid() { + int id = defineSelfCustomTarget(); + given().log() + .all() + .when() + .pathParams(Map.of("targetId", id, "threadDumpId", "foo")) + .delete("targets/{targetId}/threaddump/{threadDumpId}") + .then() + .log() + .all() + .and() + .assertThat() + .statusCode(400); + } + + @Test + void testDownloadInvalid() { + given().log() + .all() + .when() + .get("/threaddump/download/abcd1234") + .then() + .assertThat() + .statusCode(400); + } + + @Test + void testDownloadNotFound() { + given().log() + .all() + .when() + .get("/api/v4/download/Zm9vL2Jhcg==") + .then() + .assertThat() + .statusCode(404); + } +} From b37b15d949330465ca3a33516c75f7f807cb149c Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Fri, 4 Jul 2025 19:44:25 -0400 Subject: [PATCH 15/22] test full workflow (create, list, delete) --- .../cryostat/diagnostics/ThreadDumpsTest.java | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/test/java/io/cryostat/diagnostics/ThreadDumpsTest.java b/src/test/java/io/cryostat/diagnostics/ThreadDumpsTest.java index 39db5da8e..74b126d17 100644 --- a/src/test/java/io/cryostat/diagnostics/ThreadDumpsTest.java +++ b/src/test/java/io/cryostat/diagnostics/ThreadDumpsTest.java @@ -80,6 +80,64 @@ public void testCreate() o -> jobId.equals(o.getJsonObject("message").getString("jobId"))); } + @Test + public void testCreateAndDelete() + throws InterruptedException, IOException, DeploymentException, TimeoutException { + int id = defineSelfCustomTarget(); + String jobId = + given().log() + .all() + .when() + .pathParams(Map.of("targetId", id)) + .post("targets/{targetId}/threaddump") + .then() + .log() + .all() + .and() + .assertThat() + .contentType(ContentType.JSON) + .statusCode(200) + .body("size()", Matchers.equalTo(0)) + .extract() + .body() + .asString(); + + expectWebSocketNotification( + "ThreadDumpSuccess", + o -> jobId.equals(o.getJsonObject("message").getString("jobId"))); + + String threadDumpId = + given().log() + .all() + .when() + .pathParams(Map.of("targetId", id)) + .get("targets/{targetId}/threaddump") + .then() + .log() + .all() + .and() + .assertThat() + .contentType(ContentType.JSON) + .statusCode(200) + .body("size()", Matchers.greaterThan(0)) + .extract() + .asString() + .replace("[", "") + .replace("]", ""); + + given().log() + .all() + .when() + .pathParams(Map.of("targetId", id, "threadDumpId", threadDumpId)) + .delete("targets/{targetId}/threaddump/{threadDumpId}") + .then() + .log() + .all() + .and() + .assertThat() + .statusCode(200); + } + @Test public void testListInvalid() { given().log() From 22da4a7685a234757da20c3100147259ef337982 Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Fri, 4 Jul 2025 21:21:47 -0400 Subject: [PATCH 16/22] Review feedback --- .../io/cryostat/diagnostic/Diagnostics.java | 23 ++- .../diagnostic/DiagnosticsHelper.java | 150 ++++-------------- .../LongRunningRequestGenerator.java | 2 +- src/main/resources/application.properties | 1 - 4 files changed, 44 insertions(+), 132 deletions(-) diff --git a/src/main/java/io/cryostat/diagnostic/Diagnostics.java b/src/main/java/io/cryostat/diagnostic/Diagnostics.java index 6bd557fd6..0b7335a08 100644 --- a/src/main/java/io/cryostat/diagnostic/Diagnostics.java +++ b/src/main/java/io/cryostat/diagnostic/Diagnostics.java @@ -38,6 +38,7 @@ import io.vertx.mutiny.core.eventbus.EventBus; import jakarta.annotation.security.RolesAllowed; import jakarta.inject.Inject; +import jakarta.ws.rs.BadRequestException; import jakarta.ws.rs.DELETE; import jakarta.ws.rs.GET; import jakarta.ws.rs.NotFoundException; @@ -55,7 +56,6 @@ import org.jboss.resteasy.reactive.RestResponse; import org.jboss.resteasy.reactive.RestResponse.ResponseBuilder; import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.NoSuchKeyException; @@ -114,16 +114,15 @@ public List getThreadDumps(@RestPath long targetId) { @Blocking @Path("targets/{targetId}/threaddump/{threadDumpId}") @RolesAllowed("write") - public void deleteThreadDump(@RestPath String threadDumpId) { + public void deleteThreadDump(@RestPath String threadDumpId, @RestPath long targetId) { try { log.tracev("Deleting thread dump with ID: {0}", threadDumpId); - storage.headObject( - HeadObjectRequest.builder().bucket(bucket).key(threadDumpId).build()); + helper.deleteThreadDump(threadDumpId, targetId); } catch (NoSuchKeyException e) { throw new NotFoundException(e); + } catch (BadRequestException e) { + throw e; } - storage.deleteObject( - DeleteObjectRequest.builder().bucket(bucket).key(threadDumpId).build()); } @Path("/threaddump/download/{encodedKey}") @@ -133,14 +132,11 @@ public void deleteThreadDump(@RestPath String threadDumpId) { public RestResponse handleStorageDownload( @RestPath String encodedKey, @RestQuery String query) throws URISyntaxException { Pair decodedKey = helper.decodedKey(encodedKey); - var threadDumpId = decodedKey.getValue().strip(); - log.tracev("Handling download Request for encodedKey: {0}", encodedKey); + log.tracev("Handling download Request for key: {0}", decodedKey); log.tracev("Handling download Request for query: {0}", query); - log.tracev("Decoded key: {0}", decodedKey.toString()); - log.tracev("UUID: {0}", threadDumpId); - log.tracev("Bucket: {0}", bucket); + String key = helper.threadDumpKey(decodedKey); try { - storage.headObject(HeadObjectRequest.builder().bucket(bucket).key(threadDumpId).build()) + storage.headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build()) .sdkHttpResponse(); } catch (NoSuchKeyException e) { throw new NotFoundException(e); @@ -157,8 +153,7 @@ public RestResponse handleStorageDownload( } log.tracev("Handling presigned download request for {0}", decodedKey); - GetObjectRequest getRequest = - GetObjectRequest.builder().bucket(bucket).key(threadDumpId).build(); + GetObjectRequest getRequest = GetObjectRequest.builder().bucket(bucket).key(key).build(); GetObjectPresignRequest presignRequest = GetObjectPresignRequest.builder() .signatureDuration(Duration.ofMinutes(1)) diff --git a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java index ea6efbc86..e730f9649 100644 --- a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java +++ b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java @@ -18,10 +18,8 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.UUID; @@ -44,19 +42,19 @@ import jakarta.ws.rs.BadRequestException; import jakarta.ws.rs.core.MediaType; import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.eclipse.microprofile.config.inject.ConfigProperty; import org.jboss.logging.Logger; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.GetObjectTaggingRequest; import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.S3Object; -import software.amazon.awssdk.services.s3.model.Tag; -import software.amazon.awssdk.services.s3.model.Tagging; @ApplicationScoped public class DiagnosticsHelper { @@ -82,8 +80,6 @@ public class DiagnosticsHelper { private static final String DIAGNOSTIC_BEAN_NAME = "com.sun.management:type=DiagnosticCommand"; static final String THREAD_DUMP_REQUESTED = "ThreadDumpRequested"; static final String THREAD_DUMP_DELETED = "ThreadDumpDeleted"; - private static final String META_KEY_NAME = "uuid"; - private static final String META_KEY_JVMID = "jvmId"; @Inject EventBus bus; @Inject TargetConnectionManager targetConnectionManager; @@ -112,8 +108,21 @@ public ThreadDump dumpThreads(String format, long targetId) { }); } + public void deleteThreadDump(String threadDumpID, long targetId) + throws BadRequestException, NoSuchKeyException { + String jvmId = Target.getTargetById(targetId).jvmId; + String key = threadDumpKey(jvmId, threadDumpID); + if (Objects.isNull(jvmId)) { + log.errorv("TargetId {0} failed to resolve to a jvmId", targetId); + throw new BadRequestException(); + } else { + storage.headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build()); + storage.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(key).build()); + } + } + public List getThreadDumps(long targetId) { - return listThreadDumps().stream() + return listThreadDumps(targetId).stream() .map( item -> { try { @@ -124,105 +133,35 @@ public List getThreadDumps(long targetId) { } }) .filter(Objects::nonNull) - .filter( - item -> { - log.tracev("Item jvmID: {0}", item.jvmId()); - log.tracev("Item key: {0}", item.uuid()); - log.tracev("Item download URL: {0}", item.downloadUrl()); - return Objects.equals( - Target.getTargetById(targetId).jvmId, item.jvmId()); - }) .toList(); } private ThreadDump convertObject(S3Object object) throws Exception { - String jvmId; - switch (storageMode(storageMode)) { - case StorageMode.TAGGING: - var req = - GetObjectTaggingRequest.builder().bucket(bucket).key(object.key()).build(); - var tagging = storage.getObjectTagging(req); - var list = tagging.tagSet(); - if (!tagging.hasTagSet() || list.isEmpty()) { - throw new Exception("No metadata found"); - } - var decodedList = new ArrayList>(); - list.forEach( - t -> { - var encodedKey = t.key(); - var decodedKey = - new String(base64Url.decode(encodedKey), StandardCharsets.UTF_8) - .trim(); - var encodedValue = t.value(); - var decodedValue = - new String( - base64Url.decode(encodedValue), - StandardCharsets.UTF_8) - .trim(); - decodedList.add(Pair.of(decodedKey, decodedValue)); - }); - jvmId = - decodedList.stream() - .filter(t -> t.getKey().equals("jvmId")) - .map(Pair::getValue) - .findFirst() - .orElseThrow(); - // content, jvmid, downloadurl, uuid - break; - case StorageMode.METADATA: - var headReq = HeadObjectRequest.builder().bucket(bucket).key(object.key()).build(); - var meta = storage.headObject(headReq).metadata(); - jvmId = Objects.requireNonNull(meta.get(META_KEY_JVMID)); - break; - case StorageMode.BUCKET: - var t = metadataService.get().read(object.key()).orElseThrow(); - jvmId = t.jvmId(); - break; - default: - throw new IllegalStateException(); - } + String jvmId = object.key().split("/")[0]; + String uuid = object.key().split("/")[1]; return new ThreadDump( - jvmId, - downloadUrl(jvmId, object.key()), - object.key(), - object.lastModified().toEpochMilli()); - } - - public String getThreadDumpContent(String uuid) throws IOException { - try (InputStream is = getModel(uuid)) { - return new String(is.readAllBytes(), StandardCharsets.UTF_8); - } - } - - private InputStream getModel(String name) { - var req = GetObjectRequest.builder().bucket(bucket).key(name).build(); - return storage.getObject(req); + jvmId, downloadUrl(jvmId, uuid), uuid, object.lastModified().toEpochMilli()); } public ThreadDump addThreadDump(String content, String jvmId) { String uuid = UUID.randomUUID().toString(); + log.tracev("Putting Thread dump into storage with key: {0}", threadDumpKey(jvmId, uuid)); var reqBuilder = PutObjectRequest.builder() .bucket(bucket) - .key(uuid) + .key(threadDumpKey(jvmId, uuid)) .contentType(MediaType.TEXT_PLAIN); switch (storageMode(storageMode)) { case StorageMode.TAGGING: - reqBuilder = reqBuilder.tagging(createTagging(jvmId)); - log.tracev("Putting Thread dump into storage with key: {0}", uuid); - log.tracev("jvmID: {0}", jvmId); - log.tracev("Bucket: {0}", bucket); break; case StorageMode.METADATA: - reqBuilder = - reqBuilder.metadata(Map.of(META_KEY_NAME, uuid, META_KEY_JVMID, jvmId)); break; case StorageMode.BUCKET: try { metadataService .get() .create( - uuid, + threadDumpKey(jvmId, uuid), new ThreadDump( jvmId, downloadUrl(jvmId, uuid), @@ -239,31 +178,6 @@ public ThreadDump addThreadDump(String content, String jvmId) { return new ThreadDump(jvmId, downloadUrl(jvmId, uuid), uuid, clock.now().getEpochSecond()); } - private Tagging createTagging(String jvmId) { - var map = Map.of("jvmId", jvmId); - var tags = new ArrayList(); - tags.addAll( - map.entrySet().stream() - .map( - e -> - Tag.builder() - .key( - base64Url.encodeAsString( - e.getKey() - .getBytes( - StandardCharsets - .UTF_8))) - .value( - base64Url.encodeAsString( - e.getValue() - .getBytes( - StandardCharsets - .UTF_8))) - .build()) - .toList()); - return Tagging.builder().tagSet(tags).build(); - } - public String downloadUrl(String jvmId, String filename) { return String.format( "/api/beta/diagnostics/threaddump/download/%s", encodedKey(jvmId, filename)); @@ -290,7 +204,7 @@ public InputStream getThreadDumpStream(String jvmId, String threadDumpID) { public InputStream getThreadDumpStream(String encodedKey) { Pair decodedKey = decodedKey(encodedKey); - var key = decodedKey.getValue().strip(); + var key = threadDumpKey(decodedKey); GetObjectRequest getRequest = GetObjectRequest.builder().bucket(bucket).key(key).build(); @@ -306,12 +220,16 @@ public Pair decodedKey(String encodedKey) { return Pair.of(parts[0], parts[1]); } - public List listThreadDumps() { - return storage - .listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).build()) - .contents() - .stream() - .toList(); + public List listThreadDumps(long targetId) { + var builder = ListObjectsV2Request.builder().bucket(bucket); + String jvmId = Target.getTargetById(targetId).jvmId; + if (Objects.isNull(jvmId)) { + log.errorv("TargetId {0} failed to resolve to a jvmId", targetId); + } + if (StringUtils.isNotBlank(jvmId)) { + builder = builder.prefix(jvmId); + } + return storage.listObjectsV2(builder.build()).contents(); } public static StorageMode storageMode(String name) { diff --git a/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java b/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java index 4cec65969..476b9109b 100644 --- a/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java +++ b/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java @@ -97,7 +97,7 @@ public LongRunningRequestGenerator() {} @ConsumeEvent(value = THREAD_DUMP_ADDRESS, blocking = true) @Transactional public void onMessage(ThreadDumpRequest request) { - logger.trace("Job ID: " + request.id() + " submitted."); + logger.tracev("Job ID: {0} submitted.", request.id()); try { var target = Target.getTargetById(request.targetId); var dump = diagnosticsHelper.dumpThreads(request.format, target.id); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 8852b78fd..871685cc6 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -121,7 +121,6 @@ storage.metadata.storage-mode=tagging storage.metadata.archives.storage-mode=${storage.metadata.storage-mode} storage.metadata.event-templates.storage-mode=${storage.metadata.storage-mode} storage.metadata.thread-dumps.storage-mode=${storage.metadata.storage-mode} -storage.metadata.thread-dumps.name=threaddumps storage.buckets.creation-retry.period=10s storage.buckets.archives.name=archivedrecordings storage.buckets.event-templates.name=eventtemplates From ecea0e3c1bd4a874a458363006bca52c27ebab97 Mon Sep 17 00:00:00 2001 From: Cryostat CI Date: Wed, 30 Jul 2025 19:03:37 +0000 Subject: [PATCH 17/22] chore(schema): automatic update --- schema/openapi.yaml | 136 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 135 insertions(+), 1 deletion(-) diff --git a/schema/openapi.yaml b/schema/openapi.yaml index 3fdc85b83..360ab568c 100644 --- a/schema/openapi.yaml +++ b/schema/openapi.yaml @@ -678,6 +678,18 @@ components: - CUSTOM - PRESET type: string + ThreadDump: + properties: + downloadUrl: + type: string + jvmId: + type: string + lastModified: + format: int64 + type: integer + uuid: + type: string + type: object UUID: format: uuid pattern: '[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}' @@ -720,6 +732,126 @@ paths: summary: Initiate a garbage collection on the specified target tags: - Diagnostics + /api/beta/diagnostics/targets/{targetId}/threaddump: + get: + parameters: + - in: path + name: targetId + required: true + schema: + format: int64 + type: integer + responses: + "200": + content: + application/json: + schema: + items: + $ref: '#/components/schemas/ThreadDump' + type: array + description: OK + "401": + description: Not Authorized + "403": + description: Not Allowed + security: + - SecurityScheme: + - read + summary: Get Thread Dumps + tags: + - Diagnostics + post: + parameters: + - in: path + name: targetId + required: true + schema: + format: int64 + type: integer + - in: query + name: format + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/HttpServerResponse' + required: true + responses: + "200": + content: + text/plain: + schema: + type: string + description: OK + "400": + description: Bad Request + "401": + description: Not Authorized + "403": + description: Not Allowed + security: + - SecurityScheme: + - write + summary: Thread Dump + tags: + - Diagnostics + /api/beta/diagnostics/targets/{targetId}/threaddump/{threadDumpId}: + delete: + parameters: + - in: path + name: targetId + required: true + schema: + format: int64 + type: integer + - in: path + name: threadDumpId + required: true + schema: + type: string + responses: + "204": + description: No Content + "401": + description: Not Authorized + "403": + description: Not Allowed + security: + - SecurityScheme: + - write + summary: Delete Thread Dump + tags: + - Diagnostics + /api/beta/diagnostics/threaddump/download/{encodedKey}: + get: + parameters: + - in: path + name: encodedKey + required: true + schema: + type: string + - in: query + name: query + schema: + type: string + responses: + "200": + content: + application/json: + schema: {} + description: OK + "401": + description: Not Authorized + "403": + description: Not Allowed + security: + - SecurityScheme: + - read + summary: Handle Storage Download + tags: + - Diagnostics /api/beta/fs/recordings: get: responses: @@ -956,7 +1088,9 @@ paths: "403": description: Not Allowed security: - - SecurityScheme: [] + - SecurityScheme: + - read + summary: List Report Rules tags: - Reports /api/v4.1/targets/{targetId}/reports: From 5a52688dbb15f2dc50fcdb579a9e0dad86a53485 Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Fri, 1 Aug 2025 13:18:12 -0400 Subject: [PATCH 18/22] Review feedback --- .../io/cryostat/diagnostic/Diagnostics.java | 9 +- .../cryostat/diagnostics/ThreadDumpsTest.java | 136 ++++++++++++------ 2 files changed, 100 insertions(+), 45 deletions(-) diff --git a/src/main/java/io/cryostat/diagnostic/Diagnostics.java b/src/main/java/io/cryostat/diagnostic/Diagnostics.java index 0b7335a08..44094803e 100644 --- a/src/main/java/io/cryostat/diagnostic/Diagnostics.java +++ b/src/main/java/io/cryostat/diagnostic/Diagnostics.java @@ -130,10 +130,10 @@ public void deleteThreadDump(@RestPath String threadDumpId, @RestPath long targe @Blocking @GET public RestResponse handleStorageDownload( - @RestPath String encodedKey, @RestQuery String query) throws URISyntaxException { + @RestPath String encodedKey, @RestQuery String filename) throws URISyntaxException { Pair decodedKey = helper.decodedKey(encodedKey); log.tracev("Handling download Request for key: {0}", decodedKey); - log.tracev("Handling download Request for query: {0}", query); + log.tracev("Handling download Request for query: {0}", filename); String key = helper.threadDumpKey(decodedKey); try { storage.headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build()) @@ -178,13 +178,14 @@ public RestResponse handleStorageDownload( } ResponseBuilder response = ResponseBuilder.create(RestResponse.Status.PERMANENT_REDIRECT); - if (StringUtils.isNotBlank(query)) { + if (StringUtils.isNotBlank(filename)) { response = response.header( HttpHeaders.CONTENT_DISPOSITION, String.format( "attachment; filename=\"%s\"", - new String(base64Url.decode(query), StandardCharsets.UTF_8))); + new String( + base64Url.decode(filename), StandardCharsets.UTF_8))); } return response.location(uri).build(); } diff --git a/src/test/java/io/cryostat/diagnostics/ThreadDumpsTest.java b/src/test/java/io/cryostat/diagnostics/ThreadDumpsTest.java index 74b126d17..63b8eed36 100644 --- a/src/test/java/io/cryostat/diagnostics/ThreadDumpsTest.java +++ b/src/test/java/io/cryostat/diagnostics/ThreadDumpsTest.java @@ -19,6 +19,8 @@ import java.io.IOException; import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import io.cryostat.AbstractTransactionalTestBase; @@ -57,54 +59,106 @@ public void testListNone() { public void testCreate() throws InterruptedException, IOException, DeploymentException, TimeoutException { int id = defineSelfCustomTarget(); - String jobId = - given().log() - .all() - .when() - .pathParams(Map.of("targetId", id)) - .post("targets/{targetId}/threaddump") - .then() - .log() - .all() - .and() - .assertThat() - .contentType(ContentType.JSON) - .statusCode(200) - .body("size()", Matchers.equalTo(0)) - .extract() - .body() - .asString(); + Executors.newSingleThreadScheduledExecutor() + .schedule( + () -> { + given().log() + .all() + .when() + .pathParams(Map.of("targetId", id)) + .post("targets/{targetId}/threaddump") + .then() + .log() + .all() + .and() + .assertThat() + .contentType(ContentType.JSON) + .statusCode(200) + .body("size()", Matchers.equalTo(0)) + .extract() + .body() + .asString(); + }, + 1, + TimeUnit.SECONDS); + + expectWebSocketNotification("ThreadDumpSuccess"); + } - expectWebSocketNotification( - "ThreadDumpSuccess", - o -> jobId.equals(o.getJsonObject("message").getString("jobId"))); + @Test + public void testCreateAndList() + throws IOException, DeploymentException, InterruptedException, TimeoutException { + // Check that creating a thread dump works as expected + int id = defineSelfCustomTarget(); + Executors.newSingleThreadScheduledExecutor() + .schedule( + () -> { + given().log() + .all() + .when() + .pathParams(Map.of("targetId", id)) + .post("targets/{targetId}/threaddump") + .then() + .log() + .all() + .and() + .assertThat() + .contentType(ContentType.JSON) + .statusCode(200) + .body("size()", Matchers.equalTo(0)) + .extract() + .body() + .asString(); + }, + 1, + TimeUnit.SECONDS); + + expectWebSocketNotification("ThreadDumpSuccess"); + + // Check that the listing is non empty + given().log() + .all() + .when() + .pathParams(Map.of("targetId", id)) + .get("targets/{targetId}/threaddump") + .then() + .log() + .all() + .and() + .assertThat() + .contentType(ContentType.JSON) + .statusCode(200) + .body("size()", Matchers.greaterThan(0)); } @Test public void testCreateAndDelete() throws InterruptedException, IOException, DeploymentException, TimeoutException { int id = defineSelfCustomTarget(); - String jobId = - given().log() - .all() - .when() - .pathParams(Map.of("targetId", id)) - .post("targets/{targetId}/threaddump") - .then() - .log() - .all() - .and() - .assertThat() - .contentType(ContentType.JSON) - .statusCode(200) - .body("size()", Matchers.equalTo(0)) - .extract() - .body() - .asString(); - - expectWebSocketNotification( - "ThreadDumpSuccess", - o -> jobId.equals(o.getJsonObject("message").getString("jobId"))); + Executors.newSingleThreadScheduledExecutor() + .schedule( + () -> { + given().log() + .all() + .when() + .pathParams(Map.of("targetId", id)) + .post("targets/{targetId}/threaddump") + .then() + .log() + .all() + .and() + .assertThat() + .contentType(ContentType.JSON) + .statusCode(200) + .body("size()", Matchers.equalTo(0)) + .extract() + .body() + .asString(); + }, + 1, + TimeUnit.SECONDS); + + expectWebSocketNotification("ThreadDumpSuccess"); String threadDumpId = given().log() From ccbea34d388c85290b96e4371414312015f6b866 Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Wed, 27 Aug 2025 16:54:43 -0400 Subject: [PATCH 19/22] deserialize thread dumps retrieved from agent --- .../java/io/cryostat/targets/AgentClient.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main/java/io/cryostat/targets/AgentClient.java b/src/main/java/io/cryostat/targets/AgentClient.java index 1a3a884ab..722ff8299 100644 --- a/src/main/java/io/cryostat/targets/AgentClient.java +++ b/src/main/java/io/cryostat/targets/AgentClient.java @@ -158,13 +158,15 @@ Uni invokeMBeanOperation( })) .map(HttpResponse::bodyAsBuffer) .map( - buff -> { - if (returnType.equals(String.class)) { - return (T) buff.toString(); - } - // TODO implement conditional handling based on expected returnType - return null; - }); + Unchecked.function( + buff -> { + if (returnType.equals(String.class)) { + return mapper.readValue(buff.toString(), returnType); + } + // TODO implement conditional handling based on expected + // returnType + return null; + })); } catch (JsonProcessingException e) { logger.error("invokeMBeanOperation request failed", e); return Uni.createFrom().failure(e); From 19b502598ee058e34da73480cec862b7da03b0df Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Thu, 28 Aug 2025 13:17:46 -0400 Subject: [PATCH 20/22] Generate filename for content-disposition header --- .../io/cryostat/diagnostic/Diagnostics.java | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/src/main/java/io/cryostat/diagnostic/Diagnostics.java b/src/main/java/io/cryostat/diagnostic/Diagnostics.java index 44094803e..8352c2ee5 100644 --- a/src/main/java/io/cryostat/diagnostic/Diagnostics.java +++ b/src/main/java/io/cryostat/diagnostic/Diagnostics.java @@ -146,7 +146,8 @@ public RestResponse handleStorageDownload( return ResponseBuilder.ok() .header( HttpHeaders.CONTENT_DISPOSITION, - String.format("attachment; filename=\"%s\"", decodedKey.getValue())) + String.format( + "attachment; filename=\"%s\"", generateFileName(decodedKey))) .header(HttpHeaders.CONTENT_TYPE, HttpMimeType.OCTET_STREAM.mime()) .entity(helper.getThreadDumpStream(encodedKey)) .build(); @@ -178,18 +179,30 @@ public RestResponse handleStorageDownload( } ResponseBuilder response = ResponseBuilder.create(RestResponse.Status.PERMANENT_REDIRECT); - if (StringUtils.isNotBlank(filename)) { - response = - response.header( - HttpHeaders.CONTENT_DISPOSITION, - String.format( - "attachment; filename=\"%s\"", - new String( - base64Url.decode(filename), StandardCharsets.UTF_8))); - } + response = + response.header( + HttpHeaders.CONTENT_DISPOSITION, + String.format( + "attachment; filename=\"%s\"", + filename.isBlank() + ? generateFileName(decodedKey) + : new String( + base64Url.decode(filename), + StandardCharsets.UTF_8))); return response.location(uri).build(); } + private String generateFileName(Pair decodedKey) { + String jvmId = decodedKey.getLeft(); + String uuid = decodedKey.getRight(); + Target t = Target.getTargetByJvmId(jvmId).get(); + if (Objects.isNull(t)) { + log.errorv("jvmId {0} failed to resolve to target. Defaulting to uuid.", jvmId); + return uuid; + } + return t.alias + "_" + uuid + ".thread_dump"; + } + @Path("targets/{targetId}/gc") @RolesAllowed("write") @Blocking From 42c9c31819a952f6a364cfd89b5395f2b65320cc Mon Sep 17 00:00:00 2001 From: Cryostat CI Date: Thu, 28 Aug 2025 17:23:05 +0000 Subject: [PATCH 21/22] chore(schema): automatic update --- schema/openapi.yaml | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/schema/openapi.yaml b/schema/openapi.yaml index b31cec144..c8d9877e4 100644 --- a/schema/openapi.yaml +++ b/schema/openapi.yaml @@ -743,13 +743,6 @@ paths: $ref: '#/components/schemas/ThreadDump' type: array description: OK - "401": - description: Not Authorized - "403": - description: Not Allowed - security: - - SecurityScheme: - - read summary: Get Thread Dumps tags: - Diagnostics @@ -780,13 +773,6 @@ paths: description: OK "400": description: Bad Request - "401": - description: Not Authorized - "403": - description: Not Allowed - security: - - SecurityScheme: - - write summary: Thread Dump tags: - Diagnostics @@ -807,13 +793,6 @@ paths: responses: "204": description: No Content - "401": - description: Not Authorized - "403": - description: Not Allowed - security: - - SecurityScheme: - - write summary: Delete Thread Dump tags: - Diagnostics @@ -826,7 +805,7 @@ paths: schema: type: string - in: query - name: query + name: filename schema: type: string responses: @@ -835,13 +814,6 @@ paths: application/json: schema: {} description: OK - "401": - description: Not Authorized - "403": - description: Not Allowed - security: - - SecurityScheme: - - read summary: Handle Storage Download tags: - Diagnostics From 9ca960efa0760c7133f45a0bfa549c7e4b746291 Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Thu, 28 Aug 2025 16:40:53 -0400 Subject: [PATCH 22/22] Fix bug caused by rebase --- src/main/java/io/cryostat/targets/AgentClient.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/cryostat/targets/AgentClient.java b/src/main/java/io/cryostat/targets/AgentClient.java index 3bce48855..c43b3e693 100644 --- a/src/main/java/io/cryostat/targets/AgentClient.java +++ b/src/main/java/io/cryostat/targets/AgentClient.java @@ -174,7 +174,8 @@ Uni invokeMBeanOperation( Unchecked.function( buff -> { if (returnType.equals(String.class)) { - return mapper.readValue(buff.toString(), returnType); + return mapper.readValue( + (InputStream) buff.getEntity(), returnType); } // TODO implement conditional handling based on expected // returnType