Skip to content

Commit

Permalink
Auto merge of rust-lang#129836 - jieyouxu:exp-msvc-ci-2, r=<try>
Browse files Browse the repository at this point in the history
[EXPERIMENTAL] more msvc ci debugging, do not look, cursed things inside

Messing around with msvc CI again... This time just trying to find if any processes are holding on to e.g. `miri.exe`. This is just a proof of concept, Chris probably knows how to implement this correctly lol. It uses `sysinfo` later because I got lazy and can't be bummed to also implement the find process info stuff in syscalls.

Please do not look inside as this PR contains provenance crimes and most likely UB.

This PR is not a place of honor... no highly esteemed deed is commemorated here... nothing valued is here.

What is here was dangerous and repulsive to us. This message is a warning about danger.

The danger is in a particular location... it increases towards a center... the center of danger is here... of a particular size and shape, and below us.

The danger is still present, in your time, as it was in ours.

The danger is to the body, and it can kill.

The form of the danger is an emanation of Windows API calls.

The danger is unleashed only if you substantially look at this PR. This PR is best shunned and left uninhabited.

r? ghost

try-job: x86_64-msvc-ext
  • Loading branch information
bors committed Sep 1, 2024
2 parents 1a1cc05 + b65b2d9 commit 9dd9033
Show file tree
Hide file tree
Showing 5 changed files with 282 additions and 7 deletions.
7 changes: 7 additions & 0 deletions src/bootstrap/Cargo.lock
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@ version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"

[[package]]
name = "anyhow"
version = "1.0.86"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"

[[package]]
name = "bitflags"
version = "2.6.0"
Expand All @@ -36,6 +42,7 @@ dependencies = [
name = "bootstrap"
version = "0.0.0"
dependencies = [
"anyhow",
"build_helper",
"cc",
"clap",
Expand Down
12 changes: 11 additions & 1 deletion src/bootstrap/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ build = "build.rs"
default-run = "bootstrap"

[features]
default = ["sysinfo"]
build-metrics = ["sysinfo"]
bootstrap-self-test = [] # enabled in the bootstrap unit tests

Expand Down Expand Up @@ -41,7 +42,7 @@ cc = "=1.0.97"
cmake = "=0.1.48"

build_helper = { path = "../tools/build_helper" }
clap = { version = "4.4", default-features = false, features = ["std", "usage", "help", "derive", "error-context"] }
clap = { version = "4.4", default-features = false, features = ["derive", "error-context", "help", "std", "usage"] }
clap_complete = "4.4"
fd-lock = "4.0"
home = "0.5"
Expand All @@ -62,6 +63,9 @@ toml = "0.5"
walkdir = "2.4"
xz2 = "0.1"

# EXPERIMENTAL
anyhow = "1"

# Dependencies needed by the build-metrics feature
sysinfo = { version = "0.31.2", default-features = false, optional = true, features = ["system"] }

Expand All @@ -71,13 +75,19 @@ version = "1.0.0"
[target.'cfg(windows)'.dependencies.windows]
version = "0.52"
features = [
"Wdk_Foundation",
"Wdk_Storage_FileSystem",
"Wdk_System_SystemServices",
"Win32_Foundation",
"Win32_Security",
"Win32_Storage_FileSystem",
"Win32_System_Diagnostics_Debug",
"Win32_System_IO",
"Win32_System_JobObjects",
"Win32_System_ProcessStatus",
"Win32_System_Threading",
"Win32_System_Time",
"Win32_System_WindowsProgramming",
]

[dev-dependencies]
Expand Down
108 changes: 108 additions & 0 deletions src/bootstrap/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ use crate::utils::helpers::{

mod core;
mod utils;
#[cfg(windows)]
mod windows_hacks;

pub use core::builder::PathSet;
pub use core::config::flags::{Flags, Subcommand};
Expand Down Expand Up @@ -1663,12 +1665,118 @@ Executed at: {executed_at}"#,
if src == dst {
return;
}

#[cfg(windows)]
{
// HACK(jieyouxu): let's see what's holding up. Note that this is not robost to TOCTOU
// races where the process was holding on to the file when calling `remove_file` but
// released immediately after before gathering process IDs holding the file.
if fs::symlink_metadata(dst).is_ok() {
let mut process_ids = windows_hacks::process_ids_using_file(dst).unwrap();
process_ids.dedup();
process_ids.sort();

if !process_ids.is_empty() {
eprintln!(
"[DEBUG] copy_link_internal: pre-remove_file: pids holding dst=`{}`: {:?}",
dst.display(),
process_ids
);
}

use sysinfo::{ProcessRefreshKind, RefreshKind, System};

let sys = System::new_with_specifics(
RefreshKind::new().with_processes(ProcessRefreshKind::everything()),
);

let mut holdups = vec![];
for (pid, process) in sys.processes() {
if process_ids.contains(&(pid.as_u32() as usize)) {
holdups.push((pid.as_u32(), process.exe().unwrap_or(Path::new(""))));
}
}

if !holdups.is_empty() {
eprintln!(
"[DEBUG] copy_link_internal: pre-remove_file: printing process names (where available) holding dst=`{}`",
dst.display()
);
for (pid, process_exe) in holdups {
eprintln!(
"[DEBUG] copy_link_internal: pre-remove_file: process holding dst=`{}`: pid={pid}, process_name={:?}",
dst.display(),
process_exe
);
}
}
}
}

if let Err(e) = fs::remove_file(dst) {
if cfg!(windows) && e.kind() != io::ErrorKind::NotFound {
// workaround for https://github.com/rust-lang/rust/issues/127126
// if removing the file fails, attempt to rename it instead.
let now = t!(SystemTime::now().duration_since(SystemTime::UNIX_EPOCH));
let _ = fs::rename(dst, format!("{}-{}", dst.display(), now.as_nanos()));

#[cfg(windows)]
{
eprintln!(
"[DEBUG]: copy_link_internal: `fs::remove_file` failed on dst=`{}`: {e}",
dst.display()
);
eprintln!("[DEBUG]: copy_link_internal: after `fs::remove_file` failed");
// HACK(jieyouxu): let's see what's holding up. Note that this is not robost to TOCTOU
// races where the process was holding on to the file when calling `remove_file` but
// released immediately after before gathering process IDs holding the file.
if fs::symlink_metadata(dst).is_ok() {
let mut process_ids = windows_hacks::process_ids_using_file(dst).unwrap();
process_ids.dedup();
process_ids.sort();

if !process_ids.is_empty() {
eprintln!(
"[DEBUG] copy_link_internal: pids holding dst=`{}`: {:?}",
dst.display(),
process_ids
);
}

use sysinfo::{ProcessRefreshKind, RefreshKind, System};

let sys = System::new_with_specifics(
RefreshKind::new().with_processes(ProcessRefreshKind::everything()),
);

let mut holdups = vec![];
for (pid, process) in sys.processes() {
if process_ids.contains(&(pid.as_u32() as usize)) {
holdups
.push((pid.as_u32(), process.exe().unwrap_or(Path::new(""))));
}
}

if holdups.is_empty() {
eprintln!(
"[DEBUG] copy_link_internal: did not find any process holding up dst=`{}`, so how did we fail?",
dst.display(),
);
} else {
eprintln!(
"[DEBUG] copy_link_internal: printing process names (where available) holding dst=`{}`",
dst.display()
);
for (pid, process_exe) in holdups {
eprintln!(
"[DEBUG] copy_link_internal: process holding dst=`{}`: pid={pid}, process_name={:?}",
dst.display(),
process_exe
);
}
}
}
}
}
}
let metadata = t!(src.symlink_metadata(), format!("src = {}", src.display()));
Expand Down
150 changes: 150 additions & 0 deletions src/bootstrap/src/windows_hacks.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
//! Experimental windows hacks to try find what the hecc is holding on to the files that cannot be
//! deleted.

// Adapted from <https://stackoverflow.com/questions/67187979/how-to-call-ntopenfile> from
// Delphi for Rust :3
// Also references <https://gist.github.com/antonioCoco/9db236d6089b4b492746f7de31b21d9d>.

// SAFETY:
// YOLO.

// Windows API naming
#![allow(nonstandard_style)]

use std::os::windows::ffi::OsStrExt;
use std::path::Path;
use std::{fs, mem};

use anyhow::{anyhow, Result};
use windows::core::PWSTR;
use windows::Wdk::Foundation::OBJECT_ATTRIBUTES;
use windows::Wdk::Storage::FileSystem::{
NtOpenFile, NtQueryInformationFile, FILE_OPEN_REPARSE_POINT,
};
use windows::Wdk::System::SystemServices::FILE_PROCESS_IDS_USING_FILE_INFORMATION;
use windows::Win32::Foundation::{
CloseHandle, HANDLE, STATUS_INFO_LENGTH_MISMATCH, UNICODE_STRING,
};
use windows::Win32::Storage::FileSystem::{
FILE_READ_ATTRIBUTES, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE,
};
use windows::Win32::System::WindowsProgramming::FILE_INFORMATION_CLASS;
use windows::Win32::System::IO::IO_STATUS_BLOCK;

/// Wraps a windows API that returns [`NTSTATUS`]:
///
/// - First convert [`NTSTATUS`] to [`HRESULT`].
/// - Then convert [`HRESULT`] into a [`WinError`] with or without optional info.
macro_rules! try_syscall {
($syscall: expr) => {{
let status = $syscall;
if status.is_err() {
::anyhow::Result::Err(::windows::core::Error::from(status.to_hresult()))?;
}
}};
($syscall: expr, $additional_info: expr) => {{
let status = $syscall;
if status.is_err() {
::anyhow::Result::Err(::windows::core::Error::new(
$syscall.into(),
$additional_info.into(),
))?;
}
}};
}

pub(crate) fn process_ids_using_file(path: &Path) -> Result<Vec<usize>> {
// Gotta have it in UTF-16LE.
let mut nt_path = {
let path = std::path::absolute(path)?;
r"\??\".encode_utf16().chain(path.as_os_str().encode_wide()).collect::<Vec<u16>>()
};

let nt_path_unicode_string = UNICODE_STRING {
Length: u16::try_from(nt_path.len() * 2)?,
MaximumLength: u16::try_from(nt_path.len() * 2)?,
Buffer: PWSTR::from_raw(nt_path.as_mut_ptr()),
};

let object_attributes = OBJECT_ATTRIBUTES {
Length: mem::size_of::<OBJECT_ATTRIBUTES>() as _,
ObjectName: &nt_path_unicode_string,
..Default::default()
};

let mut io_status = IO_STATUS_BLOCK::default();
let mut handle = HANDLE::default();

// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntifs/nf-ntifs-ntopenfile
try_syscall!(
unsafe {
NtOpenFile(
&mut handle as *mut _,
FILE_READ_ATTRIBUTES.0,
&object_attributes,
&mut io_status as *mut _,
(FILE_SHARE_READ).0,
FILE_OPEN_REPARSE_POINT.0,
)
},
"tried to open file"
);

/// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ne-wdm-_file_information_class
// Remark: apparently windows 0.52 doesn't have this or something, it appears in at least >=
// 0.53.
const FileProcessIdsUsingFileInformation: FILE_INFORMATION_CLASS = FILE_INFORMATION_CLASS(47);

// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntifs/nf-ntifs-ntqueryinformationfile
const INCREMENT: usize = 8;
let mut buf = vec![FILE_PROCESS_IDS_USING_FILE_INFORMATION::default(); INCREMENT as usize];
let mut buf_idx = 0;
let mut status = unsafe {
NtQueryInformationFile(
handle,
&mut io_status as *mut _,
buf.as_mut_ptr().cast(),
(INCREMENT * mem::size_of::<FILE_PROCESS_IDS_USING_FILE_INFORMATION>()) as u32,
FileProcessIdsUsingFileInformation,
)
};
while status == STATUS_INFO_LENGTH_MISMATCH {
buf.resize(buf.len() + INCREMENT, FILE_PROCESS_IDS_USING_FILE_INFORMATION::default());
buf_idx += INCREMENT;
status = unsafe {
NtQueryInformationFile(
handle,
&mut io_status as *mut _,
buf.as_mut_ptr()
.offset(
(buf_idx * mem::size_of::<FILE_PROCESS_IDS_USING_FILE_INFORMATION>())
as isize,
)
.cast(),
(INCREMENT * mem::size_of::<FILE_PROCESS_IDS_USING_FILE_INFORMATION>()) as u32,
FileProcessIdsUsingFileInformation,
)
};
}

let mut process_ids = vec![];

for FILE_PROCESS_IDS_USING_FILE_INFORMATION {
NumberOfProcessIdsInList,
ProcessIdList: [ptr],
} in buf
{
if NumberOfProcessIdsInList >= 1 {
// only fetch the first one
process_ids.push(unsafe {
// This is almost certaintly UB, provenance be damned
let ptr = ptr as *mut usize;
*ptr
});
}
}

try_syscall!(unsafe { CloseHandle(handle) }, "close file handle");

Ok(process_ids)
}
12 changes: 6 additions & 6 deletions src/ci/github-actions/jobs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,12 +85,12 @@ pr:
- image: mingw-check-tidy
continue_on_error: true
<<: *job-linux-4c
- image: x86_64-gnu-llvm-17
env:
ENABLE_GCC_CODEGEN: "1"
<<: *job-linux-16c
- image: x86_64-gnu-tools
<<: *job-linux-16c
#- image: x86_64-gnu-llvm-17
# env:
# ENABLE_GCC_CODEGEN: "1"
# <<: *job-linux-16c
#- image: x86_64-gnu-tools
# <<: *job-linux-16c

# Jobs that run when you perform a try build (@bors try)
# These jobs automatically inherit envs.try, to avoid repeating
Expand Down

0 comments on commit 9dd9033

Please sign in to comment.