From b8fef2cc6238274e75f50fe47dd71187a8e4d448 Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Fri, 4 Jul 2025 18:46:34 -0400 Subject: [PATCH 1/7] Rough prototype for heap dumps --- .../io/cryostat/agent/CryostatClient.java | 56 +++++++++++++++++++ .../cryostat/agent/remote/InvokeContext.java | 37 +++++++++++- 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/cryostat/agent/CryostatClient.java b/src/main/java/io/cryostat/agent/CryostatClient.java index eea17683..d9ec0d19 100644 --- a/src/main/java/io/cryostat/agent/CryostatClient.java +++ b/src/main/java/io/cryostat/agent/CryostatClient.java @@ -374,6 +374,62 @@ public CompletableFuture update( } } + public CompletableFuture pushHeapDump(int maxFiles, String uuid, Path heapDump) + throws IOException { + Instant start = Instant.now(); + String timestamp = start.truncatedTo(ChronoUnit.SECONDS).toString(); + String fileName = String.format("%s_%s.hprof", uuid, timestamp); + + HttpPost req = + new HttpPost(baseUri.resolve("/api/beta/diagnostics/heapdump/upload/" + jvmId)); + + CountingInputStream is = getRecordingInputStream(heapDump); + + MultipartEntityBuilder entityBuilder = + MultipartEntityBuilder.create() + .addPart( + FormBodyPartBuilder.create( + "recording", + new InputStreamBody( + is, + ContentType.APPLICATION_OCTET_STREAM, + fileName)) + .build()) + .addPart( + FormBodyPartBuilder.create( + "maxFiles", + new StringBody( + Integer.toString(maxFiles), + ContentType.TEXT_PLAIN)) + .build()); + req.setEntity(entityBuilder.build()); + return supply( + req, + (res) -> { + Instant finish = Instant.now(); + log.trace( + "{} {} ({} -> {}): {}/{}", + req.getMethod(), + res.getStatusLine().getStatusCode(), + fileName, + req.getURI(), + FileUtils.byteCountToDisplaySize(is.getByteCount()), + Duration.between(start, finish)); + assertOkStatus(req, res); + return (Void) null; + }) + .whenComplete( + (v, t) -> { + // Heap dump files tend to be very large, clean up after uploading + try { + Files.delete(heapDump); + } catch (IOException ioe) { + log.warn("Failed to delete heap dump: ", heapDump.toString()); + } + req.reset(); + }); + } + public CompletableFuture upload( Harvester.PushType pushType, Optional opt, diff --git a/src/main/java/io/cryostat/agent/remote/InvokeContext.java b/src/main/java/io/cryostat/agent/remote/InvokeContext.java index 6fedc664..523e4e27 100644 --- a/src/main/java/io/cryostat/agent/remote/InvokeContext.java +++ b/src/main/java/io/cryostat/agent/remote/InvokeContext.java @@ -19,12 +19,15 @@ import java.io.InputStream; import java.io.OutputStream; import java.lang.management.ManagementFactory; +import java.nio.file.Paths; import java.util.Objects; import javax.inject.Inject; import javax.management.MBeanServer; import javax.management.ObjectName; +import io.cryostat.agent.CryostatClient; + import com.fasterxml.jackson.databind.ObjectMapper; import com.sun.net.httpserver.HttpExchange; import org.apache.http.HttpStatus; @@ -37,10 +40,13 @@ class InvokeContext extends MutatingRemoteContext { private final Logger log = LoggerFactory.getLogger(getClass()); private final ObjectMapper mapper; + private CryostatClient client; + @Inject - InvokeContext(ObjectMapper mapper, Config config) { + InvokeContext(ObjectMapper mapper, Config config, CryostatClient client) { super(config); this.mapper = mapper; + this.client = client; } @Override @@ -68,6 +74,13 @@ public void handle(HttpExchange exchange) throws IOException { req.operation, req.parameters, req.signature); + // TODO: Verify if dumpHeap is blocking, if so we should split the + // invocation + // into a separate thread and listen for when it finishes + if (req.getOperation().equals("dumpHeap")) { + String fileName = req.getParameters()[0].toString(); + client.pushHeapDump(1, fileName, Paths.get(fileName)); + } if (Objects.nonNull(response)) { exchange.sendResponseHeaders(HttpStatus.SC_OK, BODY_LENGTH_UNKNOWN); try (OutputStream responseStream = exchange.getResponseBody()) { @@ -99,13 +112,33 @@ static class MBeanInvocationRequest { public Object[] parameters; public String[] signature; + private static final String HOTSPOT_DIAGNOSTIC_BEAN_NAME = + "com.sun.management:type=HotSpotDiagnostic"; + public boolean isValid() { - if (this.beanName.equals(ManagementFactory.MEMORY_MXBEAN_NAME)) { + if (this.beanName.equals(ManagementFactory.MEMORY_MXBEAN_NAME) + || this.beanName.equals(HOTSPOT_DIAGNOSTIC_BEAN_NAME)) { return true; } return false; } + public Object[] getParameters() { + return parameters; + } + + public void setParameters(Object[] parameters) { + this.parameters = parameters; + } + + public String getOperation() { + return operation; + } + + public void setOperation(String operation) { + this.operation = operation; + } + public String getBeanName() { return beanName; } From d16ce1db0502bac9669a932374ecfd79353a082a Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Fri, 29 Aug 2025 15:16:52 -0400 Subject: [PATCH 2/7] Correct rebase error --- .../java/io/cryostat/agent/remote/InvokeContext.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/main/java/io/cryostat/agent/remote/InvokeContext.java b/src/main/java/io/cryostat/agent/remote/InvokeContext.java index 952681a0..9edbe86b 100644 --- a/src/main/java/io/cryostat/agent/remote/InvokeContext.java +++ b/src/main/java/io/cryostat/agent/remote/InvokeContext.java @@ -77,7 +77,7 @@ public void handle(HttpExchange exchange) throws IOException { req.operation, req.parameters, req.signature); - + // TODO: Verify if dumpHeap is blocking, if so we should split the // invocation // into a separate thread and listen for when it finishes @@ -140,14 +140,6 @@ public void setParameters(Object[] parameters) { this.parameters = parameters; } - public String getOperation() { - return operation; - } - - public void setOperation(String operation) { - this.operation = operation; - } - public String getBeanName() { return beanName; } From 8d40ae9ccd76872cfd6683a9081bda15bf35c259 Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Fri, 29 Aug 2025 17:18:14 -0400 Subject: [PATCH 3/7] Correct uploader, debug logging --- src/main/java/io/cryostat/agent/CryostatClient.java | 6 ++---- src/main/java/io/cryostat/agent/remote/InvokeContext.java | 4 ++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/io/cryostat/agent/CryostatClient.java b/src/main/java/io/cryostat/agent/CryostatClient.java index d9ec0d19..f0c4cf69 100644 --- a/src/main/java/io/cryostat/agent/CryostatClient.java +++ b/src/main/java/io/cryostat/agent/CryostatClient.java @@ -374,11 +374,9 @@ public CompletableFuture update( } } - public CompletableFuture pushHeapDump(int maxFiles, String uuid, Path heapDump) + public CompletableFuture pushHeapDump(int maxFiles, String fileName, Path heapDump) throws IOException { Instant start = Instant.now(); - String timestamp = start.truncatedTo(ChronoUnit.SECONDS).toString(); - String fileName = String.format("%s_%s.hprof", uuid, timestamp); HttpPost req = new HttpPost(baseUri.resolve("/api/beta/diagnostics/heapdump/upload/" + jvmId)); @@ -389,7 +387,7 @@ public CompletableFuture pushHeapDump(int maxFiles, String uuid, Path heap MultipartEntityBuilder.create() .addPart( FormBodyPartBuilder.create( - "recording", + "heapDump", new InputStreamBody( is, ContentType.APPLICATION_OCTET_STREAM, diff --git a/src/main/java/io/cryostat/agent/remote/InvokeContext.java b/src/main/java/io/cryostat/agent/remote/InvokeContext.java index 9edbe86b..a5753d08 100644 --- a/src/main/java/io/cryostat/agent/remote/InvokeContext.java +++ b/src/main/java/io/cryostat/agent/remote/InvokeContext.java @@ -20,6 +20,7 @@ import java.io.OutputStream; import java.lang.management.ManagementFactory; import java.nio.file.Paths; +import java.util.Arrays; import java.util.Objects; import javax.inject.Inject; @@ -70,6 +71,9 @@ public void handle(HttpExchange exchange) throws IOException { exchange.sendResponseHeaders( HttpStatus.SC_BAD_REQUEST, BODY_LENGTH_NONE); } + log.warn( + "Dumping heap with parameters: " + + Arrays.asList(req.parameters).toString()); MBeanServer server = ManagementFactory.getPlatformMBeanServer(); Object response = server.invoke( From e897931bc3518a9e7c14b49f82cf0bb4c35ed56e Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Thu, 11 Sep 2025 13:25:48 -0400 Subject: [PATCH 4/7] Generate filename on agent side --- .../io/cryostat/agent/remote/InvokeContext.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/main/java/io/cryostat/agent/remote/InvokeContext.java b/src/main/java/io/cryostat/agent/remote/InvokeContext.java index a5753d08..7109c567 100644 --- a/src/main/java/io/cryostat/agent/remote/InvokeContext.java +++ b/src/main/java/io/cryostat/agent/remote/InvokeContext.java @@ -20,8 +20,8 @@ import java.io.OutputStream; import java.lang.management.ManagementFactory; import java.nio.file.Paths; -import java.util.Arrays; import java.util.Objects; +import java.util.UUID; import javax.inject.Inject; import javax.management.MBeanServer; @@ -67,13 +67,18 @@ public void handle(HttpExchange exchange) throws IOException { try (InputStream body = exchange.getRequestBody()) { MBeanInvocationRequest req = mapper.readValue(body, MBeanInvocationRequest.class); + String filename = ""; if (!req.isValid()) { exchange.sendResponseHeaders( HttpStatus.SC_BAD_REQUEST, BODY_LENGTH_NONE); } - log.warn( - "Dumping heap with parameters: " - + Arrays.asList(req.parameters).toString()); + + if (req.getOperation().equals("dumpHeap")) { + String jvmId = req.getParameters()[0].toString(); + filename = jvmId + "-" + UUID.randomUUID().toString() + ".hprof"; + req.parameters[0] = filename; + } + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); Object response = server.invoke( @@ -86,8 +91,8 @@ public void handle(HttpExchange exchange) throws IOException { // invocation // into a separate thread and listen for when it finishes if (req.getOperation().equals("dumpHeap")) { - String fileName = req.getParameters()[0].toString(); - client.pushHeapDump(1, fileName, Paths.get(fileName)); + log.warn("Calling pushHeapDump"); + client.pushHeapDump(1, filename, Paths.get(filename)); } if (Objects.nonNull(response)) { From 7ea11c5f65ef6cc0c191c83c3bc878a59d2b8f92 Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Thu, 11 Sep 2025 18:19:16 -0400 Subject: [PATCH 5/7] Retrieve own alias --- .../java/io/cryostat/agent/remote/InvokeContext.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/io/cryostat/agent/remote/InvokeContext.java b/src/main/java/io/cryostat/agent/remote/InvokeContext.java index 7109c567..60cc7912 100644 --- a/src/main/java/io/cryostat/agent/remote/InvokeContext.java +++ b/src/main/java/io/cryostat/agent/remote/InvokeContext.java @@ -27,6 +27,7 @@ import javax.management.MBeanServer; import javax.management.ObjectName; +import io.cryostat.agent.ConfigModule; import io.cryostat.agent.CryostatClient; import com.fasterxml.jackson.databind.ObjectMapper; @@ -67,15 +68,15 @@ public void handle(HttpExchange exchange) throws IOException { try (InputStream body = exchange.getRequestBody()) { MBeanInvocationRequest req = mapper.readValue(body, MBeanInvocationRequest.class); - String filename = ""; + String filename = + config.getValue(ConfigModule.CRYOSTAT_AGENT_APP_NAME, String.class); if (!req.isValid()) { exchange.sendResponseHeaders( HttpStatus.SC_BAD_REQUEST, BODY_LENGTH_NONE); } if (req.getOperation().equals("dumpHeap")) { - String jvmId = req.getParameters()[0].toString(); - filename = jvmId + "-" + UUID.randomUUID().toString() + ".hprof"; + filename += "-" + UUID.randomUUID().toString() + ".hprof"; req.parameters[0] = filename; } @@ -103,7 +104,7 @@ public void handle(HttpExchange exchange) throws IOException { } else { exchange.sendResponseHeaders(HttpStatus.SC_ACCEPTED, BODY_LENGTH_NONE); } - } catch (Exception e) { + } catch (Throwable e) { log.error("mbean serialization failure", e); exchange.sendResponseHeaders(HttpStatus.SC_BAD_GATEWAY, BODY_LENGTH_NONE); } From 9ce034a10cd18bf20fdd1868b08b70481f87638d Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Mon, 15 Sep 2025 15:59:23 -0400 Subject: [PATCH 6/7] Send request ID back with heap dump upload, correct catch --- .../io/cryostat/agent/CryostatClient.java | 18 +++++++++--------- .../cryostat/agent/remote/InvokeContext.java | 19 +++++++++++++------ 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/main/java/io/cryostat/agent/CryostatClient.java b/src/main/java/io/cryostat/agent/CryostatClient.java index f0c4cf69..9dba3cc1 100644 --- a/src/main/java/io/cryostat/agent/CryostatClient.java +++ b/src/main/java/io/cryostat/agent/CryostatClient.java @@ -374,10 +374,12 @@ public CompletableFuture update( } } - public CompletableFuture pushHeapDump(int maxFiles, String fileName, Path heapDump) - throws IOException { + public CompletableFuture pushHeapDump(Path heapDump, String requestId) + throws IOException, IllegalArgumentException { Instant start = Instant.now(); - + if (heapDump.toFile().getName().isBlank()) { + throw new IllegalArgumentException("Failed to generate heap dump"); + } HttpPost req = new HttpPost(baseUri.resolve("/api/beta/diagnostics/heapdump/upload/" + jvmId)); @@ -391,14 +393,12 @@ public CompletableFuture pushHeapDump(int maxFiles, String fileName, Path new InputStreamBody( is, ContentType.APPLICATION_OCTET_STREAM, - fileName)) + heapDump.toFile().getName())) .build()) .addPart( FormBodyPartBuilder.create( - "maxFiles", - new StringBody( - Integer.toString(maxFiles), - ContentType.TEXT_PLAIN)) + "jobId", + new StringBody(requestId, ContentType.TEXT_PLAIN)) .build()); req.setEntity(entityBuilder.build()); return supply( @@ -409,7 +409,7 @@ public CompletableFuture pushHeapDump(int maxFiles, String fileName, Path "{} {} ({} -> {}): {}/{}", req.getMethod(), res.getStatusLine().getStatusCode(), - fileName, + heapDump.getFileName().toString(), req.getURI(), FileUtils.byteCountToDisplaySize(is.getByteCount()), Duration.between(start, finish)); diff --git a/src/main/java/io/cryostat/agent/remote/InvokeContext.java b/src/main/java/io/cryostat/agent/remote/InvokeContext.java index 60cc7912..e511b25b 100644 --- a/src/main/java/io/cryostat/agent/remote/InvokeContext.java +++ b/src/main/java/io/cryostat/agent/remote/InvokeContext.java @@ -88,12 +88,10 @@ public void handle(HttpExchange exchange) throws IOException { req.parameters, req.signature); - // TODO: Verify if dumpHeap is blocking, if so we should split the - // invocation - // into a separate thread and listen for when it finishes if (req.getOperation().equals("dumpHeap")) { - log.warn("Calling pushHeapDump"); - client.pushHeapDump(1, filename, Paths.get(filename)); + // Send the request Id back with the heap dump so the server + // can match it with the open requests. + client.pushHeapDump(Paths.get(filename), req.getRequestId()); } if (Objects.nonNull(response)) { @@ -104,7 +102,7 @@ public void handle(HttpExchange exchange) throws IOException { } else { exchange.sendResponseHeaders(HttpStatus.SC_ACCEPTED, BODY_LENGTH_NONE); } - } catch (Throwable e) { + } catch (Exception e) { log.error("mbean serialization failure", e); exchange.sendResponseHeaders(HttpStatus.SC_BAD_GATEWAY, BODY_LENGTH_NONE); } @@ -126,6 +124,7 @@ static class MBeanInvocationRequest { public String operation; public Object[] parameters; public String[] signature; + public String requestId; private static final String HOTSPOT_DIAGNOSTIC_BEAN_NAME = "com.sun.management:type=HotSpotDiagnostic"; @@ -165,5 +164,13 @@ public String getOperation() { public void setOperation(String operation) { this.operation = operation; } + + public void setRequestId(String requestId) { + this.requestId = requestId; + } + + public String getRequestId() { + return requestId; + } } } From 883cc24765cc1aa61d8d0e4f3b62d6c7d0d47e02 Mon Sep 17 00:00:00 2001 From: jmatsuok Date: Mon, 15 Sep 2025 17:15:43 -0400 Subject: [PATCH 7/7] Retrieve jobId from parameters, clean up catch --- .../cryostat/agent/remote/InvokeContext.java | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/main/java/io/cryostat/agent/remote/InvokeContext.java b/src/main/java/io/cryostat/agent/remote/InvokeContext.java index e511b25b..95dd8ba0 100644 --- a/src/main/java/io/cryostat/agent/remote/InvokeContext.java +++ b/src/main/java/io/cryostat/agent/remote/InvokeContext.java @@ -24,8 +24,12 @@ import java.util.UUID; import javax.inject.Inject; +import javax.management.InstanceNotFoundException; +import javax.management.MBeanException; import javax.management.MBeanServer; +import javax.management.MalformedObjectNameException; import javax.management.ObjectName; +import javax.management.ReflectionException; import io.cryostat.agent.ConfigModule; import io.cryostat.agent.CryostatClient; @@ -68,6 +72,7 @@ public void handle(HttpExchange exchange) throws IOException { try (InputStream body = exchange.getRequestBody()) { MBeanInvocationRequest req = mapper.readValue(body, MBeanInvocationRequest.class); + String requestId = (String) req.parameters[2]; String filename = config.getValue(ConfigModule.CRYOSTAT_AGENT_APP_NAME, String.class); if (!req.isValid()) { @@ -78,6 +83,9 @@ public void handle(HttpExchange exchange) throws IOException { if (req.getOperation().equals("dumpHeap")) { filename += "-" + UUID.randomUUID().toString() + ".hprof"; req.parameters[0] = filename; + // Job ID is passed along in parameters[2] + Object[] parameters = {req.parameters[0], req.parameters[1]}; + req.parameters = parameters; } MBeanServer server = ManagementFactory.getPlatformMBeanServer(); @@ -91,7 +99,7 @@ public void handle(HttpExchange exchange) throws IOException { if (req.getOperation().equals("dumpHeap")) { // Send the request Id back with the heap dump so the server // can match it with the open requests. - client.pushHeapDump(Paths.get(filename), req.getRequestId()); + client.pushHeapDump(Paths.get(filename), requestId); } if (Objects.nonNull(response)) { @@ -102,7 +110,11 @@ public void handle(HttpExchange exchange) throws IOException { } else { exchange.sendResponseHeaders(HttpStatus.SC_ACCEPTED, BODY_LENGTH_NONE); } - } catch (Exception e) { + } catch (InstanceNotFoundException + | IOException + | MBeanException + | MalformedObjectNameException + | ReflectionException e) { log.error("mbean serialization failure", e); exchange.sendResponseHeaders(HttpStatus.SC_BAD_GATEWAY, BODY_LENGTH_NONE); } @@ -124,7 +136,6 @@ static class MBeanInvocationRequest { public String operation; public Object[] parameters; public String[] signature; - public String requestId; private static final String HOTSPOT_DIAGNOSTIC_BEAN_NAME = "com.sun.management:type=HotSpotDiagnostic"; @@ -164,13 +175,5 @@ public String getOperation() { public void setOperation(String operation) { this.operation = operation; } - - public void setRequestId(String requestId) { - this.requestId = requestId; - } - - public String getRequestId() { - return requestId; - } } }