Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions projects/clr/hipamd/src/hrr/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ endif()
target_sources(amdhip64 PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/hip_capture.cpp
${CMAKE_CURRENT_SOURCE_DIR}/hip_capture_metadata.cpp
${CMAKE_CURRENT_SOURCE_DIR}/hip_capture_scrub.cpp
${CMAKE_CURRENT_SOURCE_DIR}/hip_capture_writer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/hip_capture_generated.cpp
)
Expand Down
14 changes: 12 additions & 2 deletions projects/clr/hipamd/src/hrr/hip_capture.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@

#include "hip_capture.h"
#include "hip_capture_metadata.h"
#include "hip_capture_scrub.h"
#include "hip_capture_writer.h"

// hrr_api_args.h — for hrr_args_* struct types and hrr_api_id_t enum
Expand Down Expand Up @@ -1436,6 +1437,15 @@ static size_t compute_bundle_size(const void* blob) {
return static_cast<size_t>(end);
}

// Stores a fat binary with the build paths and compiler build of its code
// objects zeroed; the application's own copy is left alone.
static hrr_cap::Hash128 write_fat_binary_blob(const void* blob, size_t blob_size) {
std::vector<uint8_t> scrubbed(static_cast<const uint8_t*>(blob),
static_cast<const uint8_t*>(blob) + blob_size);
hrr_cap::scrub::scrub_code_object(scrubbed.data(), scrubbed.size());
return hrr_cap::writer::write_blob(scrubbed.data(), scrubbed.size());
}

void** capture___hipRegisterFatBinary(const void* data) {
void** r = g_real_compiler_table.__hipRegisterFatBinary_fn(data);
// Shim is only installed when capture is active — no hip_capture_enabled() check needed.
Expand Down Expand Up @@ -1464,7 +1474,7 @@ void** capture___hipRegisterFatBinary(const void* data) {
a.ret = reinterpret_cast<uint64_t>(r);
a.blob_size = static_cast<uint64_t>(blob_size);
if (blob && blob_size > 0) {
auto h = hrr_cap::writer::write_blob(blob, blob_size);
auto h = write_fat_binary_blob(blob, blob_size);
a.blob_hash_lo = h.lo;
a.blob_hash_hi = h.hi;
}
Expand Down Expand Up @@ -2622,7 +2632,7 @@ static void record_fat_binary_blob(const void* blob_ptr) {
hrr_args___hipRegisterFatBinary a{};
a.ret = 0; // handle not meaningful at init time
a.blob_size = static_cast<uint64_t>(blob_size);
auto h = hrr_cap::writer::write_blob(blob_ptr, blob_size);
auto h = write_fat_binary_blob(blob_ptr, blob_size);
a.blob_hash_lo = h.lo;
a.blob_hash_hi = h.hi;
hrr_cap::writer::write_event_raw(HRR_API_HIPREGISTERFATBINARY, &a.hdr, sizeof(a));
Expand Down
26 changes: 5 additions & 21 deletions projects/clr/hipamd/src/hrr/hip_capture_metadata.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -88,22 +88,6 @@ std::string version_string_from_int(int version) {
return std::to_string(major) + "." + std::to_string(minor) + "." + std::to_string(patch);
}

std::string bytes_to_hex(const char* bytes, size_t len) {
static constexpr char kHex[] = "0123456789abcdef";
std::string result;
result.resize(len * 2);
for (size_t i = 0; i < len; ++i) {
const auto v = static_cast<unsigned char>(bytes[i]);
result[2 * i] = kHex[v >> 4];
result[2 * i + 1] = kHex[v & 0xf];
}
return result;
}

std::string uuid_to_hex(const hipUUID& uuid) {
return bytes_to_hex(uuid.bytes, sizeof(uuid.bytes));
}

void append_prop_fields(std::ostringstream& os, const hipDeviceProp_t& prop) {
os << " \"properties\": {\n"
<< " \"name\": " << quote(bounded_string(prop.name, sizeof(prop.name))) << ",\n"
Expand All @@ -123,11 +107,11 @@ void append_prop_fields(std::ostringstream& os, const hipDeviceProp_t& prop) {
<< " \"compute_mode\": " << prop.computeMode << ",\n"
<< " \"compute_capability\": " << quote(std::to_string(prop.major) + "." +
std::to_string(prop.minor)) << ",\n"
<< " \"pci\": " << quote(std::to_string(prop.pciDomainID) + ":" +
std::to_string(prop.pciBusID) + ":" +
std::to_string(prop.pciDeviceID)) << ",\n"
<< " \"uuid\": " << quote(uuid_to_hex(prop.uuid)) << ",\n"
<< " \"luid\": " << quote(bytes_to_hex(prop.luid, sizeof(prop.luid))) << ",\n"
// Left blank: they name the individual accelerator, and with it the machine
// that captured, and replay needs none of them.
<< " \"pci\": \"\",\n"
<< " \"uuid\": \"\",\n"
<< " \"luid\": \"\",\n"
<< " \"luid_device_node_mask\": " << prop.luidDeviceNodeMask << "\n"
<< " }";
}
Expand Down
119 changes: 119 additions & 0 deletions projects/clr/hipamd/src/hrr/hip_capture_scrub.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
/*
* Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
*
* SPDX-License-Identifier: MIT
*/

#include "hip_capture_scrub.h"

#include <cstring>

// Field offsets are spelled out rather than taken from <elf.h>, which Windows
// does not have and whose Elf64_* typedefs collide with CLR's own ELF headers.
// The image is the application's memory, so every read is bounds-checked and
// copied out with memcpy, whatever its alignment.

namespace hrr_cap {
namespace scrub {

namespace {

constexpr size_t kElfHeaderSize = 64;
constexpr size_t kSectionHeaderSize = 64;
constexpr uint8_t kElfClass64 = 2;
constexpr uint8_t kElfDataLsb = 1;
constexpr uint32_t kShtNobits = 8;
constexpr uint64_t kShfAlloc = 0x2;

constexpr char kBundleMagic[] = "__CLANG_OFFLOAD_BUNDLE__";
constexpr size_t kBundleMagicLen = sizeof(kBundleMagic) - 1;
constexpr size_t kBundleEntryHeaderSize = 3 * sizeof(uint64_t);

template <typename T>
T read(const uint8_t* p, size_t at) {
T v;
std::memcpy(&v, p + at, sizeof(v));
return v;
}

// [offset, offset + size) lies inside [0, limit).
bool in_range(uint64_t offset, uint64_t size, size_t limit) {
return offset <= limit && size <= limit - offset;
}

bool has_prefix(const char* s, size_t len, const char* prefix) {
const size_t n = std::strlen(prefix);
return len >= n && std::memcmp(s, prefix, n) == 0;
}

bool is_scrubbed_section(const char* name, size_t len) {
return (len == 8 && std::memcmp(name, ".comment", 8) == 0) || has_prefix(name, len, ".debug");
}

size_t scrub_elf(uint8_t* image, size_t size) {
if (size < kElfHeaderSize || std::memcmp(image, "\x7f" "ELF", 4) != 0 ||
image[4] != kElfClass64 || image[5] != kElfDataLsb)
return 0;
const uint64_t shoff = read<uint64_t>(image, 0x28);
const uint16_t shentsize = read<uint16_t>(image, 0x3a);
const uint16_t shnum = read<uint16_t>(image, 0x3c);
const uint16_t shstrndx = read<uint16_t>(image, 0x3e);
// shnum == 0 also covers extended section numbering, which code objects never need.
if (shentsize != kSectionHeaderSize || shnum == 0 || shstrndx >= shnum ||
!in_range(shoff, uint64_t{shnum} * kSectionHeaderSize, size))
return 0;

const uint8_t* shstr_hdr = image + shoff + size_t{shstrndx} * kSectionHeaderSize;
const uint64_t names_off = read<uint64_t>(shstr_hdr, 24);
const uint64_t names_size = read<uint64_t>(shstr_hdr, 32);
if (!in_range(names_off, names_size, size)) return 0;
const char* names = reinterpret_cast<const char*>(image + names_off);

size_t zeroed = 0;
for (uint16_t i = 0; i < shnum; ++i) {
const uint8_t* sh = image + shoff + size_t{i} * kSectionHeaderSize;
const uint32_t name = read<uint32_t>(sh, 0);
const uint32_t type = read<uint32_t>(sh, 4);
const uint64_t flags = read<uint64_t>(sh, 8);
const uint64_t offset = read<uint64_t>(sh, 24);
const uint64_t sec_size = read<uint64_t>(sh, 32);
if (name >= names_size || type == kShtNobits || (flags & kShfAlloc) != 0) continue;
const size_t name_len = strnlen(names + name, names_size - name);
if (!is_scrubbed_section(names + name, name_len)) continue;
Comment thread
andraghetti marked this conversation as resolved.
Resolved
if (sec_size == 0 || !in_range(offset, sec_size, size)) continue;
std::memset(image + offset, 0, sec_size);
++zeroed;
Comment thread
andraghetti marked this conversation as resolved.
Resolved
}
return zeroed;
}

size_t scrub_bundle(uint8_t* image, size_t size) {
size_t pos = kBundleMagicLen;
if (size < pos + sizeof(uint64_t)) return 0;
const uint64_t count = read<uint64_t>(image, pos);
pos += sizeof(uint64_t);
size_t zeroed = 0;
for (uint64_t i = 0; i < count; ++i) {
if (size - pos < kBundleEntryHeaderSize) break;
const uint64_t offset = read<uint64_t>(image, pos);
const uint64_t entry_size = read<uint64_t>(image, pos + 8);
const uint64_t id_size = read<uint64_t>(image, pos + 16);
pos += kBundleEntryHeaderSize;
if (id_size > size - pos) break;
pos += id_size;
if (in_range(offset, entry_size, size)) zeroed += scrub_elf(image + offset, entry_size);
}
return zeroed;
}

} // namespace

size_t scrub_code_object(uint8_t* image, size_t size) {
if (!image) return 0;
if (size >= kBundleMagicLen && std::memcmp(image, kBundleMagic, kBundleMagicLen) == 0)
return scrub_bundle(image, size);
return scrub_elf(image, size);
}

} // namespace scrub
} // namespace hrr_cap
24 changes: 24 additions & 0 deletions projects/clr/hipamd/src/hrr/hip_capture_scrub.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/*
* Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
*
* SPDX-License-Identifier: MIT
*/
#pragma once

#include <cstddef>
#include <cstdint>

namespace hrr_cap {
namespace scrub {

// Zeroes, in place, the contents of the .comment and .debug* sections of a code
// object about to be stored: an ELF64 image, or every entry of an uncompressed
// clang offload bundle. Those sections carry the compiler build, absolute build
// paths and source file names; the GPU never loads them, so replay is unchanged.
// Sizes and offsets stay as they are, so a bundle's entry table stays valid.
// A compressed bundle, or anything that does not parse, is left as it is.
// Returns how many sections were zeroed.
size_t scrub_code_object(uint8_t* image, size_t size);

} // namespace scrub
} // namespace hrr_cap
11 changes: 9 additions & 2 deletions projects/clr/hipamd/src/hrr/hip_capture_writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
#include "hip_capture_writer.h"
#include "hip_capture.h"
#include "hip_capture_metadata.h"
#include "hip_capture_scrub.h"

#include "os/os.hpp" // amd::Os::timeNanos()
#include "utils/debug.hpp" // LogPrintfError, LogPrintfWarning, LogPrintfInfo
Expand Down Expand Up @@ -1638,7 +1639,13 @@ Hash128 write_code_object(const void* image, size_t image_size) {
if (g_events_fd < 0 || g_events_finalized) return {}; // not open, or past the trailer
}

Hash128 h = hash_buffer(image, image_size);
// The stored bytes, and so the hash, are the image without its build paths
// and compiler build (see hip_capture_scrub.h).
std::vector<uint8_t> scrubbed(static_cast<const uint8_t*>(image),
static_cast<const uint8_t*>(image) + image_size);
scrub::scrub_code_object(scrubbed.data(), scrubbed.size());

Hash128 h = hash_buffer(scrubbed.data(), image_size);
Comment thread
andraghetti marked this conversation as resolved.
Resolved
char hex[33];
hash_hex(h, hex);
std::string key = std::string("co:") + hex; // namespace to match playback load_code_object key
Expand All @@ -1658,7 +1665,7 @@ Hash128 write_code_object(const void* image, size_t image_size) {
ClaimedWrite claim(key);

std::string path = g_output_dir + "/code_objects/" + hex + ".hsaco";
const bool ok = atomic_write_file(path, image, image_size);
const bool ok = atomic_write_file(path, scrubbed.data(), image_size);
if (!ok) LogPrintfWarning("[HRR capture] Failed to write code object %s", hex);
claim.done(ok);
return h;
Expand Down
6 changes: 5 additions & 1 deletion projects/hrr/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,11 @@ capture.hrr/
- **code_objects/** — code objects, content-addressed by hash (device images
extracted by the runtime, for `hipModuleLoad` too); a launch
records the hash so kernels sharing a name (Triton emits many `triton_`)
still resolve to the code object they came from
still resolve to the code object they came from. Their `.comment` and
`.debug*` sections, which hold the compiler build and build paths, are
zeroed before they are stored, as are those of uncompressed fat binaries
- **manifest.json** — counts, versions and device properties; the GPU's PCI
address, UUID and LUID are written blank
- **Complete: NO** — original run crashed before clean shutdown; reader still recovers complete events

Code objects read back from the runtime must start with `\x7fELF`,
Expand Down
8 changes: 4 additions & 4 deletions projects/hrr/skills/enable-recording/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -124,19 +124,19 @@ An archive with several `pid-*` directories is one archive. Send the whole direc

## 4. Share it, knowing what is in it

**There is no scrubbing, anonymisation or randomisation option today.** Say that plainly before anyone asks whether the archive can be sanitised, because the answer shapes what they do next and there is no flag that changes it.
**There is no scrubbing, anonymisation or randomisation option for the data today.** Say that plainly before anyone asks whether the archive can be sanitised, because the answer shapes what they do next and there is no flag that changes it. Capture does drop two things on its own, with no flag: the build paths and compiler build inside the code objects, and the GPU's hardware identifiers.

What the archive holds, per process, and it is worth reading out to whoever has to approve the transfer:

- **`events.bin`**: every successful HIP call the process made, with its arguments: sizes, device addresses, stream handles, kernel names.
- **`blobs/`**: the host buffers those calls carried, byte for byte and in the clear. For a serving stack that means prompts and generated text; for training, the data and the weights that were copied to the device.
- **`code_objects/`**: the compiled GPU kernels the process loaded, which is the application's own binary code and a separate disclosure question from customer data.
- **`manifest.json`**: process and parent process ids, event and blob counts, the clean-shutdown flag, the HIP and comgr versions that captured, and the device's name, architecture, PCI location and UUID.
- **`code_objects/`**: the compiled GPU kernels the process loaded, which is the application's own binary code and a separate disclosure question from customer data. Their `.comment` and `.debug*` sections are zeroed, here and in the fat binaries under `blobs/`, so the compiler build, source file names and build paths are gone, except in compressed (`CCOB`) fat binaries, which are stored as the application had them. Kernel names stay.
- **`manifest.json`**: process and parent process ids, event and blob counts, the clean-shutdown flag, the HIP and comgr versions that captured, and the device's name, architecture, memory size and clocks. Its PCI location, UUID and LUID are written blank.
- **`regions/*.hrrr`**: the framework allocator's own layout, when a producer ran alongside capture.

Apart from the few exceptions in step 2, failed HIP calls are not recorded, and nothing that never crossed a HIP call is. The archive is exactly as sensitive as the data the workload handled, so where it may go has to be agreed before it is uploaded.

`verify --json` gives each process's counts and sizes in a form you can attach to the request; the device details are in each `manifest.json`. For a multi-process archive, `hrr-playback --info` against the archive root prints a table covering every process at once. The cross-check in `verify` reports on one process, which is enough to judge the capture and not enough for a disclosure inventory.
`verify --json` gives each process's counts and sizes in a form you can attach to the request, and says whether any hardware identifier or code object build information is left (`hardware_ids`, `build_info_files`, and `unchecked_fat_binaries` for the compressed ones it cannot read); the device details are in each `manifest.json`. For a multi-process archive, `hrr-playback --info` against the archive root prints a table covering every process at once. The cross-check in `verify` reports on one process, which is enough to judge the capture and not enough for a disclosure inventory.

If the content is the obstacle, three things exist and none of them is scrubbing. Re-capture the same failure driven by synthetic input, if it still reproduces. Replay the archive yourself and send only the finding, which is a few lines and carries no buffers. Or agree the handling of a production-data transfer before uploading, because that is what this is.

Expand Down
Loading
Loading