diff --git a/schema/openapi.yaml b/schema/openapi.yaml index b07065bef..c8d9877e4 100644 --- a/schema/openapi.yaml +++ b/schema/openapi.yaml @@ -678,6 +678,18 @@ components: - CUSTOM - PRESET type: string + ThreadDump: + properties: + downloadUrl: + type: string + jvmId: + type: string + lastModified: + format: int64 + type: integer + uuid: + type: string + type: object UUID: format: uuid pattern: '[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}' @@ -713,6 +725,98 @@ paths: summary: Initiate a garbage collection on the specified target tags: - Diagnostics + /api/beta/diagnostics/targets/{targetId}/threaddump: + get: + parameters: + - in: path + name: targetId + required: true + schema: + format: int64 + type: integer + responses: + "200": + content: + application/json: + schema: + items: + $ref: '#/components/schemas/ThreadDump' + type: array + description: OK + summary: Get Thread Dumps + tags: + - Diagnostics + post: + parameters: + - in: path + name: targetId + required: true + schema: + format: int64 + type: integer + - in: query + name: format + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/HttpServerResponse' + required: true + responses: + "200": + content: + text/plain: + schema: + type: string + description: OK + "400": + description: Bad Request + summary: Thread Dump + tags: + - Diagnostics + /api/beta/diagnostics/targets/{targetId}/threaddump/{threadDumpId}: + delete: + parameters: + - in: path + name: targetId + required: true + schema: + format: int64 + type: integer + - in: path + name: threadDumpId + required: true + schema: + type: string + responses: + "204": + description: No Content + summary: Delete Thread Dump + tags: + - Diagnostics + /api/beta/diagnostics/threaddump/download/{encodedKey}: + get: + parameters: + - in: path + name: encodedKey + required: true + schema: + type: string + - in: query + name: filename + schema: + type: string + responses: + "200": + content: + application/json: + schema: {} + description: OK + summary: Handle Storage Download + tags: + - Diagnostics /api/beta/fs/recordings: get: responses: diff --git a/smoketest.bash b/smoketest.bash index 8d4a2c437..d9f014712 100755 --- a/smoketest.bash +++ b/smoketest.bash @@ -17,7 +17,7 @@ PULL_IMAGES=${PULL_IMAGES:-true} KEEP_VOLUMES=${KEEP_VOLUMES:-false} OPEN_TABS=${OPEN_TABS:-false} -PRECREATE_BUCKETS=${PRECREATE_BUCKETS:-archivedrecordings,archivedreports,eventtemplates,probes} +PRECREATE_BUCKETS=${PRECREATE_BUCKETS:-archivedrecordings,archivedreports,eventtemplates,probes,threaddumps} LOG_LEVEL=0 CRYOSTAT_HTTP_HOST=${CRYOSTAT_HTTP_HOST:-cryostat} diff --git a/src/main/java/io/cryostat/ConfigProperties.java b/src/main/java/io/cryostat/ConfigProperties.java index cbb4df903..21434e78c 100644 --- a/src/main/java/io/cryostat/ConfigProperties.java +++ b/src/main/java/io/cryostat/ConfigProperties.java @@ -28,6 +28,9 @@ public class ConfigProperties { "storage.buckets.event-templates.name"; public static final String AWS_BUCKET_NAME_PROBE_TEMPLATES = "storage.buckets.probe-templates.name"; + public static final String AWS_BUCKET_NAME_THREAD_DUMPS = "storage.buckets.thread-dumps.name"; + public static final String AWS_METADATA_PREFIX_THREAD_DUMPS = + "storage.metadata.prefix.thread-dumps"; public static final String AWS_METADATA_PREFIX_RECORDINGS = "storage.metadata.prefix.recordings"; public static final String AWS_METADATA_PREFIX_EVENT_TEMPLATES = @@ -67,6 +70,9 @@ public class ConfigProperties { public static final String STORAGE_PRESIGNED_DOWNLOADS_ENABLED = "storage.presigned-downloads.enabled"; + public static final String STORAGE_METADATA_THREAD_DUMPS_STORAGE_MODE = + "storage.metadata.thread-dumps.storage-mode"; + public static final String CUSTOM_TEMPLATES_DIR = "templates-dir"; public static final String PRESET_TEMPLATES_DIR = "preset-templates-dir"; public static final String PROBE_TEMPLATES_DIR = "probe-templates-dir"; diff --git a/src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java b/src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java new file mode 100644 index 000000000..857a984cc --- /dev/null +++ b/src/main/java/io/cryostat/diagnostic/BucketedDiagnosticsMetadataService.java @@ -0,0 +1,130 @@ +/* + * Copyright The Cryostat Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cryostat.diagnostic; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import io.cryostat.ConfigProperties; +import io.cryostat.StorageBuckets; +import io.cryostat.diagnostic.Diagnostics.ThreadDump; +import io.cryostat.recordings.ArchivedRecordingMetadataService; +import io.cryostat.util.CRUDService; +import io.cryostat.util.HttpMimeType; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.quarkus.arc.lookup.LookupIfProperty; +import io.quarkus.runtime.StartupEvent; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Observes; +import jakarta.inject.Inject; +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.jboss.logging.Logger; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; + +@ApplicationScoped +@LookupIfProperty( + name = ConfigProperties.STORAGE_METADATA_THREAD_DUMPS_STORAGE_MODE, + stringValue = ArchivedRecordingMetadataService.METADATA_STORAGE_MODE_BUCKET) +public class BucketedDiagnosticsMetadataService + implements CRUDService { + + @ConfigProperty(name = ConfigProperties.STORAGE_METADATA_THREAD_DUMPS_STORAGE_MODE) + String storageMode; + + @ConfigProperty(name = ConfigProperties.AWS_BUCKET_NAME_METADATA) + String bucket; + + @ConfigProperty(name = ConfigProperties.AWS_METADATA_PREFIX_THREAD_DUMPS) + String prefix; + + @Inject S3Client storage; + @Inject StorageBuckets buckets; + + @Inject ObjectMapper mapper; + + @Inject Logger logger; + + void onStart(@Observes StartupEvent evt) { + if (!ArchivedRecordingMetadataService.METADATA_STORAGE_MODE_BUCKET.equals(storageMode)) { + return; + } + buckets.createIfNecessary(bucket); + } + + @Override + public List list() throws IOException { + var builder = ListObjectsV2Request.builder().bucket(bucket).prefix(prefix); + var objs = storage.listObjectsV2(builder.build()).contents(); + return objs.stream() + .map( + t -> { + // TODO this entails a remote file read over the network and then some + // minor processing of the received file. More time will be spent + // retrieving the data than processing it, so this should be + // parallelized. + try { + return read(t.key()).orElseThrow(); + } catch (IOException e) { + logger.error(e); + return null; + } + }) + .filter(Objects::nonNull) + .toList(); + } + + @Override + public void create(String k, ThreadDump threadDump) throws IOException { + storage.putObject( + PutObjectRequest.builder() + .bucket(bucket) + .key(prefix(k)) + .contentType(HttpMimeType.PLAINTEXT.mime()) + .build(), + RequestBody.fromBytes(mapper.writeValueAsBytes(threadDump))); + } + + @Override + public Optional read(String k) throws IOException { + try (var stream = + new BufferedInputStream( + storage.getObject( + GetObjectRequest.builder() + .bucket(bucket) + .key(prefix(k)) + .build()))) { + return Optional.of(mapper.readValue(stream, ThreadDump.class)); + } + } + + @Override + public void delete(String k) throws IOException { + storage.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(prefix(k)).build()); + } + + private String prefix(String key) { + return String.format("%s/%s", prefix, key); + } +} diff --git a/src/main/java/io/cryostat/diagnostic/Diagnostics.java b/src/main/java/io/cryostat/diagnostic/Diagnostics.java index b729922f3..8352c2ee5 100644 --- a/src/main/java/io/cryostat/diagnostic/Diagnostics.java +++ b/src/main/java/io/cryostat/diagnostic/Diagnostics.java @@ -15,23 +15,195 @@ */ package io.cryostat.diagnostic; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; + +import io.cryostat.ConfigProperties; +import io.cryostat.Producers; +import io.cryostat.recordings.LongRunningRequestGenerator; +import io.cryostat.recordings.LongRunningRequestGenerator.ThreadDumpRequest; import io.cryostat.targets.Target; import io.cryostat.targets.TargetConnectionManager; +import io.cryostat.util.HttpMimeType; import io.smallrye.common.annotation.Blocking; +import io.smallrye.common.annotation.Identifier; +import io.vertx.core.http.HttpServerResponse; +import io.vertx.mutiny.core.eventbus.EventBus; import jakarta.annotation.security.RolesAllowed; import jakarta.inject.Inject; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.NotFoundException; import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; +import jakarta.ws.rs.core.HttpHeaders; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.eclipse.microprofile.config.inject.ConfigProperty; import org.eclipse.microprofile.openapi.annotations.Operation; +import org.jboss.logging.Logger; import org.jboss.resteasy.reactive.RestPath; +import org.jboss.resteasy.reactive.RestQuery; +import org.jboss.resteasy.reactive.RestResponse; +import org.jboss.resteasy.reactive.RestResponse.ResponseBuilder; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; +import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; -@Path("/api/beta/diagnostics/targets/{targetId}") +@Path("/api/beta/diagnostics/") public class Diagnostics { @Inject TargetConnectionManager targetConnectionManager; + @Inject S3Client storage; + @Inject S3Presigner presigner; + @Inject Logger log; + @Inject LongRunningRequestGenerator generator; + + @Inject + @Identifier(Producers.BASE64_URL) + Base64 base64Url; + + @ConfigProperty(name = ConfigProperties.AWS_BUCKET_NAME_THREAD_DUMPS) + String bucket; + + @ConfigProperty(name = ConfigProperties.STORAGE_PRESIGNED_DOWNLOADS_ENABLED) + boolean presignedDownloadsEnabled; - @Path("/gc") + @ConfigProperty(name = ConfigProperties.STORAGE_EXT_URL) + Optional externalStorageUrl; + + @Inject EventBus bus; + @Inject DiagnosticsHelper helper; + + @Path("targets/{targetId}/threaddump") + @RolesAllowed("write") + @POST + public String threadDump( + HttpServerResponse response, @RestPath long targetId, @RestQuery String format) { + log.tracev("Creating new thread dump request for target: {0}", targetId); + ThreadDumpRequest request = + new ThreadDumpRequest(UUID.randomUUID().toString(), targetId, format); + response.endHandler( + (e) -> bus.publish(LongRunningRequestGenerator.THREAD_DUMP_ADDRESS, request)); + return request.id(); + } + + @Path("targets/{targetId}/threaddump") + @RolesAllowed("read") + @Blocking + @GET + public List getThreadDumps(@RestPath long targetId) { + log.tracev("Fetching thread dumps for target: {0}", targetId); + return helper.getThreadDumps(targetId); + } + + @DELETE + @Blocking + @Path("targets/{targetId}/threaddump/{threadDumpId}") + @RolesAllowed("write") + public void deleteThreadDump(@RestPath String threadDumpId, @RestPath long targetId) { + try { + log.tracev("Deleting thread dump with ID: {0}", threadDumpId); + helper.deleteThreadDump(threadDumpId, targetId); + } catch (NoSuchKeyException e) { + throw new NotFoundException(e); + } catch (BadRequestException e) { + throw e; + } + } + + @Path("/threaddump/download/{encodedKey}") + @RolesAllowed("read") + @Blocking + @GET + public RestResponse handleStorageDownload( + @RestPath String encodedKey, @RestQuery String filename) throws URISyntaxException { + Pair decodedKey = helper.decodedKey(encodedKey); + log.tracev("Handling download Request for key: {0}", decodedKey); + log.tracev("Handling download Request for query: {0}", filename); + String key = helper.threadDumpKey(decodedKey); + try { + storage.headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build()) + .sdkHttpResponse(); + } catch (NoSuchKeyException e) { + throw new NotFoundException(e); + } + + if (!presignedDownloadsEnabled) { + return ResponseBuilder.ok() + .header( + HttpHeaders.CONTENT_DISPOSITION, + String.format( + "attachment; filename=\"%s\"", generateFileName(decodedKey))) + .header(HttpHeaders.CONTENT_TYPE, HttpMimeType.OCTET_STREAM.mime()) + .entity(helper.getThreadDumpStream(encodedKey)) + .build(); + } + + log.tracev("Handling presigned download request for {0}", decodedKey); + GetObjectRequest getRequest = GetObjectRequest.builder().bucket(bucket).key(key).build(); + GetObjectPresignRequest presignRequest = + GetObjectPresignRequest.builder() + .signatureDuration(Duration.ofMinutes(1)) + .getObjectRequest(getRequest) + .build(); + PresignedGetObjectRequest presignedRequest = presigner.presignGetObject(presignRequest); + URI uri = presignedRequest.url().toURI(); + if (externalStorageUrl.isPresent()) { + String extUrl = externalStorageUrl.get(); + if (StringUtils.isNotBlank(extUrl)) { + URI extUri = new URI(extUrl); + uri = + new URI( + extUri.getScheme(), + extUri.getAuthority(), + URI.create(String.format("%s/%s", extUri.getPath(), uri.getPath())) + .normalize() + .getPath(), + uri.getQuery(), + uri.getFragment()); + } + } + ResponseBuilder response = + ResponseBuilder.create(RestResponse.Status.PERMANENT_REDIRECT); + response = + response.header( + HttpHeaders.CONTENT_DISPOSITION, + String.format( + "attachment; filename=\"%s\"", + filename.isBlank() + ? generateFileName(decodedKey) + : new String( + base64Url.decode(filename), + StandardCharsets.UTF_8))); + return response.location(uri).build(); + } + + private String generateFileName(Pair decodedKey) { + String jvmId = decodedKey.getLeft(); + String uuid = decodedKey.getRight(); + Target t = Target.getTargetByJvmId(jvmId).get(); + if (Objects.isNull(t)) { + log.errorv("jvmId {0} failed to resolve to target. Defaulting to uuid.", jvmId); + return uuid; + } + return t.alias + "_" + uuid + ".thread_dump"; + } + + @Path("targets/{targetId}/gc") @RolesAllowed("write") @Blocking @POST @@ -49,4 +221,13 @@ public void gc(@RestPath long targetId) { conn.invokeMBeanOperation( "java.lang:type=Memory", "gc", null, null, Void.class)); } + + public record ThreadDump(String jvmId, String downloadUrl, String uuid, long lastModified) { + + public ThreadDump { + Objects.requireNonNull(jvmId); + Objects.requireNonNull(downloadUrl); + Objects.requireNonNull(uuid); + } + } } diff --git a/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java new file mode 100644 index 000000000..e730f9649 --- /dev/null +++ b/src/main/java/io/cryostat/diagnostic/DiagnosticsHelper.java @@ -0,0 +1,241 @@ +/* + * Copyright The Cryostat Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cryostat.diagnostic; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.UUID; + +import io.cryostat.ConfigProperties; +import io.cryostat.Producers; +import io.cryostat.StorageBuckets; +import io.cryostat.diagnostic.Diagnostics.ThreadDump; +import io.cryostat.libcryostat.sys.Clock; +import io.cryostat.recordings.ArchivedRecordingMetadataService.StorageMode; +import io.cryostat.targets.Target; +import io.cryostat.targets.TargetConnectionManager; + +import io.quarkus.runtime.StartupEvent; +import io.smallrye.common.annotation.Identifier; +import io.vertx.mutiny.core.eventbus.EventBus; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Observes; +import jakarta.enterprise.inject.Instance; +import jakarta.inject.Inject; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.core.MediaType; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.jboss.logging.Logger; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Object; + +@ApplicationScoped +public class DiagnosticsHelper { + + @ConfigProperty(name = ConfigProperties.AWS_BUCKET_NAME_THREAD_DUMPS) + String bucket; + + @ConfigProperty(name = ConfigProperties.STORAGE_METADATA_STORAGE_MODE) + String storageMode; + + @Inject + @Identifier(Producers.BASE64_URL) + Base64 base64Url; + + @Inject S3Client storage; + @Inject Logger log; + @Inject Clock clock; + + @Inject Instance metadataService; + + private static final String DUMP_THREADS = "threadPrint"; + private static final String DUMP_THREADS_TO_FIlE = "threadDumpToFile"; + private static final String DIAGNOSTIC_BEAN_NAME = "com.sun.management:type=DiagnosticCommand"; + static final String THREAD_DUMP_REQUESTED = "ThreadDumpRequested"; + static final String THREAD_DUMP_DELETED = "ThreadDumpDeleted"; + + @Inject EventBus bus; + @Inject TargetConnectionManager targetConnectionManager; + @Inject StorageBuckets buckets; + + void onStart(@Observes StartupEvent evt) { + log.tracev("Creating thread dump bucket: {0}", bucket); + buckets.createIfNecessary(bucket); + } + + public ThreadDump dumpThreads(String format, long targetId) { + if (!(format.equals(DUMP_THREADS) || format.equals(DUMP_THREADS_TO_FIlE))) { + throw new BadRequestException(); + } + log.tracev( + "Thread Dump request received for Target: {0} with format: {1}", targetId, format); + Object[] params = new Object[1]; + String[] signature = new String[] {String[].class.getName()}; + return targetConnectionManager.executeConnectedTask( + Target.getTargetById(targetId), + conn -> { + String content = + conn.invokeMBeanOperation( + DIAGNOSTIC_BEAN_NAME, format, params, signature, String.class); + return addThreadDump(content, Target.getTargetById(targetId).jvmId); + }); + } + + public void deleteThreadDump(String threadDumpID, long targetId) + throws BadRequestException, NoSuchKeyException { + String jvmId = Target.getTargetById(targetId).jvmId; + String key = threadDumpKey(jvmId, threadDumpID); + if (Objects.isNull(jvmId)) { + log.errorv("TargetId {0} failed to resolve to a jvmId", targetId); + throw new BadRequestException(); + } else { + storage.headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build()); + storage.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(key).build()); + } + } + + public List getThreadDumps(long targetId) { + return listThreadDumps(targetId).stream() + .map( + item -> { + try { + return convertObject(item); + } catch (Exception e) { + log.error(e); + return null; + } + }) + .filter(Objects::nonNull) + .toList(); + } + + private ThreadDump convertObject(S3Object object) throws Exception { + String jvmId = object.key().split("/")[0]; + String uuid = object.key().split("/")[1]; + return new ThreadDump( + jvmId, downloadUrl(jvmId, uuid), uuid, object.lastModified().toEpochMilli()); + } + + public ThreadDump addThreadDump(String content, String jvmId) { + String uuid = UUID.randomUUID().toString(); + log.tracev("Putting Thread dump into storage with key: {0}", threadDumpKey(jvmId, uuid)); + var reqBuilder = + PutObjectRequest.builder() + .bucket(bucket) + .key(threadDumpKey(jvmId, uuid)) + .contentType(MediaType.TEXT_PLAIN); + switch (storageMode(storageMode)) { + case StorageMode.TAGGING: + break; + case StorageMode.METADATA: + break; + case StorageMode.BUCKET: + try { + metadataService + .get() + .create( + threadDumpKey(jvmId, uuid), + new ThreadDump( + jvmId, + downloadUrl(jvmId, uuid), + uuid, + clock.now().getEpochSecond())); + break; + } catch (IOException ioe) { + log.warnv("Exception thrown while adding thread dump to storage: {0}", ioe); + } + default: + throw new IllegalStateException(); + } + storage.putObject(reqBuilder.build(), RequestBody.fromString(content)); + return new ThreadDump(jvmId, downloadUrl(jvmId, uuid), uuid, clock.now().getEpochSecond()); + } + + public String downloadUrl(String jvmId, String filename) { + return String.format( + "/api/beta/diagnostics/threaddump/download/%s", encodedKey(jvmId, filename)); + } + + public String encodedKey(String jvmId, String uuid) { + Objects.requireNonNull(jvmId); + Objects.requireNonNull(uuid); + return base64Url.encodeAsString( + (threadDumpKey(jvmId, uuid)).getBytes(StandardCharsets.UTF_8)); + } + + public String threadDumpKey(String jvmId, String uuid) { + return (jvmId + "/" + uuid).strip(); + } + + public String threadDumpKey(Pair pair) { + return threadDumpKey(pair.getKey(), pair.getValue()); + } + + public InputStream getThreadDumpStream(String jvmId, String threadDumpID) { + return getThreadDumpStream(encodedKey(jvmId, threadDumpID)); + } + + public InputStream getThreadDumpStream(String encodedKey) { + Pair decodedKey = decodedKey(encodedKey); + var key = threadDumpKey(decodedKey); + + GetObjectRequest getRequest = GetObjectRequest.builder().bucket(bucket).key(key).build(); + + return storage.getObject(getRequest); + } + + public Pair decodedKey(String encodedKey) { + String key = new String(base64Url.decode(encodedKey), StandardCharsets.UTF_8); + String[] parts = key.split("/"); + if (parts.length != 2) { + throw new IllegalArgumentException(); + } + return Pair.of(parts[0], parts[1]); + } + + public List listThreadDumps(long targetId) { + var builder = ListObjectsV2Request.builder().bucket(bucket); + String jvmId = Target.getTargetById(targetId).jvmId; + if (Objects.isNull(jvmId)) { + log.errorv("TargetId {0} failed to resolve to a jvmId", targetId); + } + if (StringUtils.isNotBlank(jvmId)) { + builder = builder.prefix(jvmId); + } + return storage.listObjectsV2(builder.build()).contents(); + } + + public static StorageMode storageMode(String name) { + return Arrays.asList(StorageMode.values()).stream() + .filter(s -> s.name().equalsIgnoreCase(name)) + .findFirst() + .orElseThrow(); + } +} diff --git a/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java b/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java index 43bd50d81..c8278b066 100644 --- a/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java +++ b/src/main/java/io/cryostat/recordings/LongRunningRequestGenerator.java @@ -22,6 +22,7 @@ import io.cryostat.ConfigProperties; import io.cryostat.core.reports.InterruptibleReportGenerator.AnalysisResult; +import io.cryostat.diagnostic.DiagnosticsHelper; import io.cryostat.recordings.ArchivedRecordings.ArchivedRecording; import io.cryostat.reports.AnalysisReportAggregator; import io.cryostat.reports.ReportsService; @@ -66,17 +67,23 @@ public class LongRunningRequestGenerator { public static final String ARCHIVED_REPORT_COMPLETE_ADDRESS = "io.cryostat.recording.LongRunningRequestGenerator.ArchivedReportComplete"; + public static final String THREAD_DUMP_ADDRESS = + "io.cryostat.recordings.LongRunningRequestGenerator.ThreadDump"; + private static final String ARCHIVE_RECORDING_SUCCESS = "ArchiveRecordingSuccess"; private static final String ARCHIVE_RECORDING_FAIL = "ArchiveRecordingFailure"; private static final String GRAFANA_UPLOAD_SUCCESS = "GrafanaUploadSuccess"; private static final String GRAFANA_UPLOAD_FAIL = "GrafanaUploadFailure"; private static final String REPORT_SUCCESS = "ReportSuccess"; private static final String REPORT_FAILURE = "ReportFailure"; + private static final String THREAD_DUMP_SUCCESS = "ThreadDumpSuccess"; + private static final String THREAD_DUMP_FAILURE = "ThreadDumpFailure"; @Inject Logger logger; @Inject private EventBus bus; @Inject private RecordingHelper recordingHelper; @Inject private ReportsService reportsService; + @Inject private DiagnosticsHelper diagnosticsHelper; @Inject AnalysisReportAggregator analysisReportAggregator; @ConfigProperty(name = ConfigProperties.CONNECTIONS_FAILED_TIMEOUT) @@ -87,6 +94,33 @@ public class LongRunningRequestGenerator { public LongRunningRequestGenerator() {} + @ConsumeEvent(value = THREAD_DUMP_ADDRESS, blocking = true) + @Transactional + public void onMessage(ThreadDumpRequest request) { + logger.tracev("Job ID: {0} submitted.", request.id()); + try { + var target = Target.getTargetById(request.targetId); + var dump = diagnosticsHelper.dumpThreads(request.format, target.id); + bus.publish( + MessagingServer.class.getName(), + new Notification( + THREAD_DUMP_SUCCESS, + Map.of( + "jobId", + request.id(), + "targetId", + dump.jvmId(), + "downloadUrl", + dump.downloadUrl()))); + } catch (Exception e) { + logger.warn("Failed to dump threads"); + bus.publish( + MessagingServer.class.getName(), + new Notification(THREAD_DUMP_FAILURE, Map.of("jobId", request.id()))); + throw new CompletionException(e); + } + } + @ConsumeEvent(value = ARCHIVE_REQUEST_ADDRESS, blocking = true) @Transactional public ArchivedRecording onMessage(ArchiveRequest request) { @@ -347,4 +381,12 @@ public record ArchivedReportCompletion( Objects.requireNonNull(report); } } + + public record ThreadDumpRequest(String id, long targetId, String format) { + public ThreadDumpRequest { + Objects.requireNonNull(id); + Objects.requireNonNull(targetId); + Objects.requireNonNull(format); + } + } } diff --git a/src/main/java/io/cryostat/targets/AgentClient.java b/src/main/java/io/cryostat/targets/AgentClient.java index ea5eee6f1..c43b3e693 100644 --- a/src/main/java/io/cryostat/targets/AgentClient.java +++ b/src/main/java/io/cryostat/targets/AgentClient.java @@ -137,6 +137,7 @@ Uni mbeanMetrics() { }); } + @SuppressWarnings("unchecked") Uni invokeMBeanOperation( String beanName, String operation, @@ -170,15 +171,16 @@ Uni invokeMBeanOperation( } })) .map( - r -> { - try (r; - // var is = r.getEntity(); - ) { - // TODO implement conditional handling based on expected - // returnType - return null; - } - }); + Unchecked.function( + buff -> { + if (returnType.equals(String.class)) { + return mapper.readValue( + (InputStream) buff.getEntity(), returnType); + } + // TODO implement conditional handling based on expected + // returnType + return null; + })); } catch (JsonProcessingException e) { logger.error("invokeMBeanOperation request failed", e); return Uni.createFrom().failure(e); diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index fcdd73337..f7890abe8 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -37,7 +37,7 @@ quarkus.datasource.devservices.db-name=quarkus # !!! quarkus.s3.devservices.enabled=true -quarkus.s3.devservices.buckets=archivedrecordings,archivedreports,eventtemplates,probes +quarkus.s3.devservices.buckets=archivedrecordings,archivedreports,eventtemplates,probes,threaddumps # FIXME the following overrides should not be required, but currently seem to help with testcontainers reliability quarkus.aws.devservices.localstack.image-name=quay.io/hazelcast_cloud/localstack:4.1.1 quarkus.aws.devservices.localstack.container-properties.START_WEB=0 diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index bff694af7..3c2a14cd1 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -129,13 +129,16 @@ storage.presigned-downloads.enabled=false storage.metadata.storage-mode=tagging storage.metadata.archives.storage-mode=${storage.metadata.storage-mode} storage.metadata.event-templates.storage-mode=${storage.metadata.storage-mode} +storage.metadata.thread-dumps.storage-mode=${storage.metadata.storage-mode} storage.buckets.creation-retry.period=10s storage.buckets.archives.name=archivedrecordings storage.buckets.event-templates.name=eventtemplates storage.buckets.probe-templates.name=probes +storage.buckets.thread-dumps.name=threaddumps storage.buckets.metadata.name=metadata storage.metadata.prefix.recordings=${storage.buckets.archives.name} storage.metadata.prefix.event-templates=${storage.buckets.event-templates.name} +storage.metadata.prefix.thread-dumps=${storage.buckets.thread-dumps.name} quarkus.quinoa.build-dir=dist quarkus.quinoa.enable-spa-routing=true diff --git a/src/test/java/io/cryostat/diagnostics/ThreadDumpsTest.java b/src/test/java/io/cryostat/diagnostics/ThreadDumpsTest.java new file mode 100644 index 000000000..63b8eed36 --- /dev/null +++ b/src/test/java/io/cryostat/diagnostics/ThreadDumpsTest.java @@ -0,0 +1,248 @@ +/* + * Copyright The Cryostat Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cryostat.diagnostics; + +import static io.restassured.RestAssured.given; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import io.cryostat.AbstractTransactionalTestBase; +import io.cryostat.diagnostic.Diagnostics; + +import io.quarkus.test.common.http.TestHTTPEndpoint; +import io.quarkus.test.junit.QuarkusTest; +import io.restassured.http.ContentType; +import jakarta.websocket.DeploymentException; +import org.hamcrest.Matchers; +import org.junit.Test; + +@QuarkusTest +@TestHTTPEndpoint(Diagnostics.class) +public class ThreadDumpsTest extends AbstractTransactionalTestBase { + + @Test + public void testListNone() { + int id = defineSelfCustomTarget(); + given().log() + .all() + .when() + .pathParams(Map.of("targetId", id)) + .get("targets/{targetId}/threaddump") + .then() + .log() + .all() + .and() + .assertThat() + .contentType(ContentType.JSON) + .statusCode(200) + .body("size()", Matchers.equalTo(0)); + } + + @Test + public void testCreate() + throws InterruptedException, IOException, DeploymentException, TimeoutException { + int id = defineSelfCustomTarget(); + Executors.newSingleThreadScheduledExecutor() + .schedule( + () -> { + given().log() + .all() + .when() + .pathParams(Map.of("targetId", id)) + .post("targets/{targetId}/threaddump") + .then() + .log() + .all() + .and() + .assertThat() + .contentType(ContentType.JSON) + .statusCode(200) + .body("size()", Matchers.equalTo(0)) + .extract() + .body() + .asString(); + }, + 1, + TimeUnit.SECONDS); + + expectWebSocketNotification("ThreadDumpSuccess"); + } + + @Test + public void testCreateAndList() + throws IOException, DeploymentException, InterruptedException, TimeoutException { + // Check that creating a thread dump works as expected + int id = defineSelfCustomTarget(); + Executors.newSingleThreadScheduledExecutor() + .schedule( + () -> { + given().log() + .all() + .when() + .pathParams(Map.of("targetId", id)) + .post("targets/{targetId}/threaddump") + .then() + .log() + .all() + .and() + .assertThat() + .contentType(ContentType.JSON) + .statusCode(200) + .body("size()", Matchers.equalTo(0)) + .extract() + .body() + .asString(); + }, + 1, + TimeUnit.SECONDS); + + expectWebSocketNotification("ThreadDumpSuccess"); + + // Check that the listing is non empty + given().log() + .all() + .when() + .pathParams(Map.of("targetId", id)) + .get("targets/{targetId}/threaddump") + .then() + .log() + .all() + .and() + .assertThat() + .contentType(ContentType.JSON) + .statusCode(200) + .body("size()", Matchers.greaterThan(0)); + } + + @Test + public void testCreateAndDelete() + throws InterruptedException, IOException, DeploymentException, TimeoutException { + int id = defineSelfCustomTarget(); + Executors.newSingleThreadScheduledExecutor() + .schedule( + () -> { + given().log() + .all() + .when() + .pathParams(Map.of("targetId", id)) + .post("targets/{targetId}/threaddump") + .then() + .log() + .all() + .and() + .assertThat() + .contentType(ContentType.JSON) + .statusCode(200) + .body("size()", Matchers.equalTo(0)) + .extract() + .body() + .asString(); + }, + 1, + TimeUnit.SECONDS); + + expectWebSocketNotification("ThreadDumpSuccess"); + + String threadDumpId = + given().log() + .all() + .when() + .pathParams(Map.of("targetId", id)) + .get("targets/{targetId}/threaddump") + .then() + .log() + .all() + .and() + .assertThat() + .contentType(ContentType.JSON) + .statusCode(200) + .body("size()", Matchers.greaterThan(0)) + .extract() + .asString() + .replace("[", "") + .replace("]", ""); + + given().log() + .all() + .when() + .pathParams(Map.of("targetId", id, "threadDumpId", threadDumpId)) + .delete("targets/{targetId}/threaddump/{threadDumpId}") + .then() + .log() + .all() + .and() + .assertThat() + .statusCode(200); + } + + @Test + public void testListInvalid() { + given().log() + .all() + .when() + .pathParams(Map.of("targetId", -1)) + .get("targets/{targetId}/threaddump") + .then() + .log() + .all() + .and() + .assertThat() + .contentType(ContentType.JSON) + .statusCode(400); + } + + @Test + void testDeleteInvalid() { + int id = defineSelfCustomTarget(); + given().log() + .all() + .when() + .pathParams(Map.of("targetId", id, "threadDumpId", "foo")) + .delete("targets/{targetId}/threaddump/{threadDumpId}") + .then() + .log() + .all() + .and() + .assertThat() + .statusCode(400); + } + + @Test + void testDownloadInvalid() { + given().log() + .all() + .when() + .get("/threaddump/download/abcd1234") + .then() + .assertThat() + .statusCode(400); + } + + @Test + void testDownloadNotFound() { + given().log() + .all() + .when() + .get("/api/v4/download/Zm9vL2Jhcg==") + .then() + .assertThat() + .statusCode(404); + } +}