feat(diagnostics): Add thread dumps endpoint and storage

smoketest.bash

@@ -17,7 +17,7 @@ PULL_IMAGES=${PULL_IMAGES:-true}
 KEEP_VOLUMES=${KEEP_VOLUMES:-false}
 OPEN_TABS=${OPEN_TABS:-false}
 
-PRECREATE_BUCKETS=${PRECREATE_BUCKETS:-archivedrecordings,archivedreports,eventtemplates,probes}
+PRECREATE_BUCKETS=${PRECREATE_BUCKETS:-archivedrecordings,archivedreports,eventtemplates,probes,threaddumps}
 
 LOG_LEVEL=0
 CRYOSTAT_HTTP_HOST=${CRYOSTAT_HTTP_HOST:-cryostat}

src/main/java/io/cryostat/ConfigProperties.java

@@ -16,11 +16,16 @@
 package io.cryostat;
 
 public class ConfigProperties {
+    public static final String STORAGE_METADATA_STORAGE_MODE = "storage.metadata.storage-mode";
     public static final String AWS_BUCKET_NAME_ARCHIVES = "storage.buckets.archives.name";
+    public static final String AWS_BUCKET_NAME_METADATA = "storage.buckets.metadata.name";
     public static final String AWS_BUCKET_NAME_EVENT_TEMPLATES =
             "storage.buckets.event-templates.name";
     public static final String AWS_BUCKET_NAME_PROBE_TEMPLATES =
             "storage.buckets.probe-templates.name";
+    public static final String AWS_BUCKET_NAME_THREAD_DUMPS = "storage.buckets.thread-dumps.name";
+    public static final String AWS_METADATA_PREFIX_THREAD_DUMPS =
+            "storage.metadata.prefix.thread-dumps";
 
     public static final String CONTAINERS_POLL_PERIOD = "cryostat.discovery.containers.poll-period";
     public static final String CONTAINERS_REQUEST_TIMEOUT =
@@ -52,6 +57,9 @@ public class ConfigProperties {
     public static final String STORAGE_PRESIGNED_DOWNLOADS_ENABLED =
             "storage.presigned-downloads.enabled";
 
+    public static final String STORAGE_METADATA_THREAD_DUMPS_STORAGE_MODE =
+            "storage.metadata.thread-dumps.storage.mode";
+
     public static final String CUSTOM_TEMPLATES_DIR = "templates-dir";
     public static final String PRESET_TEMPLATES_DIR = "preset-templates-dir";
     public static final String PROBE_TEMPLATES_DIR = "probe-templates-dir";

src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java

@@ -0,0 +1,146 @@
+/*
+ * Copyright The Cryostat Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.cryostat.diagnostic;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+
+import io.cryostat.ConfigProperties;
+import io.cryostat.StorageBuckets;
+import io.cryostat.diagnostic.Diagnostics.ThreadDump;
+import io.cryostat.util.CRUDService;
+import io.cryostat.util.HttpMimeType;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.quarkus.arc.lookup.LookupIfProperty;
+import io.quarkus.runtime.StartupEvent;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.event.Observes;
+import jakarta.inject.Inject;
+import org.eclipse.microprofile.config.inject.ConfigProperty;
+import org.jboss.logging.Logger;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+
+@ApplicationScoped
+@LookupIfProperty(
+        name = ConfigProperties.STORAGE_METADATA_THREAD_DUMPS_STORAGE_MODE,
+        stringValue = DiagnosticsHelper.METADATA_STORAGE_MODE_BUCKET)
+public class BucketedDiagnosticsMetadataService
+        implements CRUDService<String, ThreadDump, ThreadDump> {
+
+    @ConfigProperty(name = ConfigProperties.STORAGE_METADATA_THREAD_DUMPS_STORAGE_MODE)
+    String storageMode;
+
+    @ConfigProperty(name = ConfigProperties.AWS_BUCKET_NAME_METADATA)
+    String bucket;
+
+    @ConfigProperty(name = ConfigProperties.AWS_METADATA_PREFIX_THREAD_DUMPS)
+    String prefix;
+
+    @Inject S3Client storage;
+    @Inject StorageBuckets buckets;
+
+    @Inject ObjectMapper mapper;
+
+    @Inject Logger logger;
+
+    void onStart(@Observes StartupEvent evt) {
+        if (!DiagnosticsHelper.METADATA_STORAGE_MODE_BUCKET.equals(storageMode)) {
+            return;
+        }
+        buckets.createIfNecessary(bucket);
+    }
+
+    @Override
+    public List<ThreadDump> list() throws IOException {
+        var builder = ListObjectsV2Request.builder().bucket(bucket).prefix(prefix);
+        var objs = storage.listObjectsV2(builder.build()).contents();
+        return objs.stream()
+                .map(
+                        t -> {
+                            // TODO this entails a remote file read over the network and then some
+                            // minor processing of the received file. More time will be spent
+                            // retrieving the data than processing it, so this should be
+                            // parallelized.
+                            try {
+                                return read(t.key()).orElseThrow();
+                            } catch (IOException e) {
+                                logger.error(e);
+                                return null;
+                            }
+                        })
+                .filter(Objects::nonNull)
+                .toList();
+    }
+
+    @Override
+    public void create(String k, ThreadDump threadDump) throws IOException {
+        storage.putObject(
+                PutObjectRequest.builder()
+                        .bucket(bucket)
+                        .key(prefix(k))
+                        .contentType(HttpMimeType.JFC.mime())
+                        .build(),
+                RequestBody.fromBytes(mapper.writeValueAsBytes(ThreadDumpMeta.from(threadDump))));
+    }
+
+    @Override
+    public Optional<ThreadDump> read(String k) throws IOException {
+        try (var stream =
+                new BufferedInputStream(
+                        storage.getObject(
+                                GetObjectRequest.builder()
+                                        .bucket(bucket)
+                                        .key(prefix(k))
+                                        .build()))) {
+            return Optional.of(mapper.readValue(stream, ThreadDumpMeta.class))
+                    .map(ThreadDumpMeta::asThreadDump);
+        }
+    }
+
+    @Override
+    public void delete(String k) throws IOException {
+        storage.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(prefix(k)).build());
+    }
+
+    private String prefix(String key) {
+        return String.format("%s/%s", prefix, key);
+    }
+
+    // just a thin serialization adapter. Jackson ObjectMapper complains about not being able to
+    // instantiate the Template type directly.
+    static record ThreadDumpMeta(String uuid, String jvmId, String content, String downloadUrl) {
+        static ThreadDumpMeta from(ThreadDump threadDump) {
+            return new ThreadDumpMeta(
+                    threadDump.uuid(),
+                    threadDump.jvmId(),
+                    threadDump.content(),
+                    threadDump.downloadUrl());
+        }
+
+        ThreadDump asThreadDump() {
+            return new ThreadDump(content, jvmId, downloadUrl, uuid);
+        }
+    }
+}

src/main/java/io/cryostat/diagnostic/Diagnostics.java

@@ -15,22 +15,185 @@
  */
 package io.cryostat.diagnostic;
 
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.UUID;
+
+import io.cryostat.ConfigProperties;
+import io.cryostat.Producers;
+import io.cryostat.recordings.LongRunningRequestGenerator;
+import io.cryostat.recordings.LongRunningRequestGenerator.ThreadDumpRequest;
 import io.cryostat.targets.Target;
 import io.cryostat.targets.TargetConnectionManager;
+import io.cryostat.util.HttpMimeType;
 
 import io.smallrye.common.annotation.Blocking;
+import io.vertx.core.http.HttpServerResponse;
+import io.vertx.mutiny.core.eventbus.EventBus;
 import jakarta.annotation.security.RolesAllowed;
 import jakarta.inject.Inject;
+import jakarta.inject.Named;
+import jakarta.ws.rs.DELETE;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.NotFoundException;
 import jakarta.ws.rs.POST;
 import jakarta.ws.rs.Path;
+import jakarta.ws.rs.core.HttpHeaders;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.eclipse.microprofile.config.inject.ConfigProperty;
+import org.jboss.logging.Logger;
 import org.jboss.resteasy.reactive.RestPath;
+import org.jboss.resteasy.reactive.RestQuery;
+import org.jboss.resteasy.reactive.RestResponse;
+import org.jboss.resteasy.reactive.RestResponse.ResponseBuilder;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
+import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
+import software.amazon.awssdk.services.s3.presigner.S3Presigner;
+import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
+import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
 
-@Path("/api/beta/diagnostics/targets/{targetId}")
+@Path("/api/beta/diagnostics/")
 public class Diagnostics {
 
     @Inject TargetConnectionManager targetConnectionManager;
+    @Inject S3Client storage;
+    @Inject S3Presigner presigner;
+    @Inject Logger log;
+    @Inject LongRunningRequestGenerator generator;
+
+    @Inject
+    @Named(Producers.BASE64_URL)
+    Base64 base64Url;
+
+    @ConfigProperty(name = ConfigProperties.AWS_BUCKET_NAME_THREAD_DUMPS)
+    String bucket;
+
+    @ConfigProperty(name = ConfigProperties.STORAGE_PRESIGNED_DOWNLOADS_ENABLED)
+    boolean presignedDownloadsEnabled;
+
+    @ConfigProperty(name = ConfigProperties.STORAGE_EXT_URL)
+    Optional<String> externalStorageUrl;
+
+    @Inject EventBus bus;
+    @Inject DiagnosticsHelper helper;
+
+    @Path("targets/{targetId}/threaddump")
+    @RolesAllowed("write")
+    @Blocking
+    @POST
+    public String threadDump(
+            HttpServerResponse response, @RestPath long targetId, @RestQuery String format) {
+        log.tracev("Creating new thread dump request for target: {0}", targetId);
+        ThreadDumpRequest request =
+                new ThreadDumpRequest(
+                        UUID.randomUUID().toString(), Long.toString(targetId), format);
+        response.endHandler(
+                (e) -> bus.publish(LongRunningRequestGenerator.THREAD_DUMP_ADDRESS, request));
+        return request.id();
+    }
+
+    @Path("targets/{targetId}/threaddump")
+    @RolesAllowed("read")
+    @Blocking
+    @GET
+    public List<ThreadDump> getThreadDumps(@RestPath long targetId) {
+        log.tracev("Fetching thread dumps for target: {0}", targetId);
+        log.tracev("Thread dumps: {0}", helper.getThreadDumps(targetId));
+        log.tracev("Storage bucket: {0}", bucket);
+        return helper.getThreadDumps(targetId);
+    }
 
-    @Path("/gc")
+    @DELETE
+    @Blocking
+    @Path("targets/{targetId}/threaddump/{threadDumpId}")
+    @RolesAllowed("write")
+    public void deleteThreadDump(@RestPath String threadDumpId) {
+        try {
+            log.tracev("Deleting thread dump with ID: {0}", threadDumpId);
+            storage.headObject(
+                    HeadObjectRequest.builder().bucket(bucket).key(threadDumpId).build());
+        } catch (NoSuchKeyException e) {
+            throw new NotFoundException(e);
+        }
+        storage.deleteObject(
+                DeleteObjectRequest.builder().bucket(bucket).key(threadDumpId).build());
+    }
+
+    @Path("/threaddump/download/{encodedKey}")
+    @RolesAllowed("read")
+    @Blocking
+    @GET
+    public RestResponse<Object> handleStorageDownload(
+            @RestPath String encodedKey, @RestQuery String query) throws URISyntaxException {
+        Pair<String, String> decodedKey = helper.decodedKey(encodedKey);
+        var threadDumpId = decodedKey.getValue().strip();
+        log.tracev("Handling download Request for encodedKey: {0}", encodedKey);
+        log.tracev("Handling download Request for query: {0}", query);
+        log.tracev("Decoded key: {0}", decodedKey.toString());
+        log.tracev("UUID: {0}", threadDumpId);
+        log.tracev("Bucket: {0}", bucket);
+        storage.headObject(HeadObjectRequest.builder().bucket(bucket).key(threadDumpId).build())
+                .sdkHttpResponse();
+
+        if (!presignedDownloadsEnabled) {
+            return ResponseBuilder.ok()
+                    .header(
+                            HttpHeaders.CONTENT_DISPOSITION,
+                            String.format("attachment; filename=\"%s\"", decodedKey.getValue()))
+                    .header(HttpHeaders.CONTENT_TYPE, HttpMimeType.OCTET_STREAM.mime())
+                    .entity(helper.getThreadDumpStream(encodedKey))
+                    .build();
+        }
+
+        log.tracev("Handling presigned download request for {0}", decodedKey);
+        GetObjectRequest getRequest =
+                GetObjectRequest.builder().bucket(bucket).key(threadDumpId).build();
+        GetObjectPresignRequest presignRequest =
+                GetObjectPresignRequest.builder()
+                        .signatureDuration(Duration.ofMinutes(1))
+                        .getObjectRequest(getRequest)
+                        .build();
+        PresignedGetObjectRequest presignedRequest = presigner.presignGetObject(presignRequest);
+        URI uri = presignedRequest.url().toURI();
+        if (externalStorageUrl.isPresent()) {
+            String extUrl = externalStorageUrl.get();
+            if (StringUtils.isNotBlank(extUrl)) {
+                URI extUri = new URI(extUrl);
+                uri =
+                        new URI(
+                                extUri.getScheme(),
+                                extUri.getAuthority(),
+                                URI.create(String.format("%s/%s", extUri.getPath(), uri.getPath()))
+                                        .normalize()
+                                        .getPath(),
+                                uri.getQuery(),
+                                uri.getFragment());
+            }
+        }
+        ResponseBuilder<Object> response =
+                ResponseBuilder.create(RestResponse.Status.PERMANENT_REDIRECT);
+        if (StringUtils.isNotBlank(query)) {
+            response =
+                    response.header(
+                            HttpHeaders.CONTENT_DISPOSITION,
+                            String.format(
+                                    "attachment; filename=\"%s\"",
+                                    new String(base64Url.decode(query), StandardCharsets.UTF_8)));
+        }
+        return response.location(uri).build();
+    }
+
+    @Path("targets/{targetId}/gc")
     @RolesAllowed("write")
     @Blocking
     @POST
@@ -41,4 +204,14 @@ public void gc(@RestPath long targetId) {
                         conn.invokeMBeanOperation(
                                 "java.lang:type=Memory", "gc", null, null, Void.class));
     }
+
+    public record ThreadDump(String content, String jvmId, String downloadUrl, String uuid) {
+
+        public ThreadDump {
+            Objects.requireNonNull(content);
+            Objects.requireNonNull(jvmId);
+            Objects.requireNonNull(downloadUrl);
+            Objects.requireNonNull(uuid);
+        }
+    }
 }