Auto merge of #107609 - pnkfelix:backport-reverts-to-fix-thin-archive…

…-reading-for-1-68-beta, r=cuviper

Backport reverts to fix thin archive reading for 1 68 beta

This is a backport of PR #107360 aimed at beta.

cc #107162, #107334 and google/shaderc-rs#133

.github/workflows/ci.yml

@@ -317,7 +317,7 @@ jobs:
               NO_DEBUG_ASSERTIONS: 1
               NO_OVERFLOW_CHECKS: 1
               DIST_REQUIRE_ALL_TOOLS: 1
-            os: macos-12-xl
+            os: macos-latest
           - name: dist-apple-various
             env:
               SCRIPT: "./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim"
@@ -328,7 +328,7 @@ jobs:
               NO_LLVM_ASSERTIONS: 1
               NO_DEBUG_ASSERTIONS: 1
               NO_OVERFLOW_CHECKS: 1
-            os: macos-12-xl
+            os: macos-latest
           - name: dist-x86_64-apple-alt
             env:
               SCRIPT: "./x.py dist bootstrap --include-default-paths"
@@ -339,7 +339,7 @@ jobs:
               NO_LLVM_ASSERTIONS: 1
               NO_DEBUG_ASSERTIONS: 1
               NO_OVERFLOW_CHECKS: 1
-            os: macos-12-xl
+            os: macos-latest
           - name: x86_64-apple-1
             env:
               SCRIPT: "./x.py --stage 2 test --exclude tests/ui --exclude tests/rustdoc --exclude tests/run-make-fulldeps"
@@ -350,7 +350,7 @@ jobs:
               NO_LLVM_ASSERTIONS: 1
               NO_DEBUG_ASSERTIONS: 1
               NO_OVERFLOW_CHECKS: 1
-            os: macos-12-xl
+            os: macos-latest
           - name: x86_64-apple-2
             env:
               SCRIPT: "./x.py --stage 2 test tests/ui tests/rustdoc tests/run-make-fulldeps"
@@ -361,7 +361,7 @@ jobs:
               NO_LLVM_ASSERTIONS: 1
               NO_DEBUG_ASSERTIONS: 1
               NO_OVERFLOW_CHECKS: 1
-            os: macos-12-xl
+            os: macos-latest
           - name: dist-aarch64-apple
             env:
               SCRIPT: "./x.py dist bootstrap --include-default-paths --stage 2"
@@ -376,7 +376,7 @@ jobs:
               NO_OVERFLOW_CHECKS: 1
               DIST_REQUIRE_ALL_TOOLS: 1
               JEMALLOC_SYS_WITH_LG_PAGE: 14
-            os: macos-12-xl
+            os: macos-latest
           - name: x86_64-msvc-1
             env:
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-profiler"

compiler/rustc_codegen_llvm/src/back/archive.rs

@@ -15,8 +15,8 @@ use crate::errors::{
 use crate::llvm::archive_ro::{ArchiveRO, Child};
 use crate::llvm::{self, ArchiveKind, LLVMMachineType, LLVMRustCOFFShortExport};
 use rustc_codegen_ssa::back::archive::{
-    get_native_object_symbols, ArArchiveBuilder, ArchiveBuildFailure, ArchiveBuilder,
-    ArchiveBuilderBuilder, UnknownArchiveKind,
+    get_native_object_symbols, try_extract_macho_fat_archive, ArArchiveBuilder,
+    ArchiveBuildFailure, ArchiveBuilder, ArchiveBuilderBuilder, UnknownArchiveKind,
 };
 
 use rustc_session::cstore::DllImport;
@@ -66,15 +66,21 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> {
         archive: &Path,
         skip: Box<dyn FnMut(&str) -> bool + 'static>,
     ) -> io::Result<()> {
-        let archive_ro = match ArchiveRO::open(archive) {
+        let mut archive = archive.to_path_buf();
+        if self.sess.target.llvm_target.contains("-apple-macosx") {
+            if let Some(new_archive) = try_extract_macho_fat_archive(&self.sess, &archive)? {
+                archive = new_archive
+            }
+        }
+        let archive_ro = match ArchiveRO::open(&archive) {
             Ok(ar) => ar,
             Err(e) => return Err(io::Error::new(io::ErrorKind::Other, e)),
         };
         if self.additions.iter().any(|ar| ar.path() == archive) {
             return Ok(());
         }
         self.additions.push(Addition::Archive {
-            path: archive.to_path_buf(),
+            path: archive,
             archive: archive_ro,
             skip: Box::new(skip),
         });
@@ -102,7 +108,9 @@ pub struct LlvmArchiveBuilderBuilder;
 
 impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder {
     fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder<'a> + 'a> {
-        if sess.target.arch == "wasm32" || sess.target.arch == "wasm64" {
+        // FIXME use ArArchiveBuilder on most targets again once reading thin archives is
+        // implemented
+        if true || sess.target.arch == "wasm32" || sess.target.arch == "wasm64" {
             Box::new(LlvmArchiveBuilder { sess, additions: Vec::new() })
         } else {
             Box::new(ArArchiveBuilder::new(sess, get_llvm_object_symbols))

compiler/rustc_codegen_ssa/src/back/archive.rs

@@ -14,7 +14,7 @@ use tempfile::Builder as TempFileBuilder;
 
 use std::error::Error;
 use std::fs::File;
-use std::io;
+use std::io::{self, Write};
 use std::path::{Path, PathBuf};
 
 // Re-exporting for rustc_codegen_llvm::back::archive
@@ -116,42 +116,51 @@ impl<'a> ArArchiveBuilder<'a> {
     }
 }
 
-fn try_filter_fat_archs<'a>(
+fn try_filter_fat_archs(
     archs: object::read::Result<&[impl FatArch]>,
     target_arch: object::Architecture,
-    archive_map_data: &'a [u8],
-) -> io::Result<Option<(&'a [u8], u64)>> {
+    archive_path: &Path,
+    archive_map_data: &[u8],
+) -> io::Result<Option<PathBuf>> {
     let archs = archs.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
 
     let desired = match archs.iter().find(|a| a.architecture() == target_arch) {
         Some(a) => a,
         None => return Ok(None),
     };
 
-    Ok(Some((
+    let (mut new_f, extracted_path) = tempfile::Builder::new()
+        .suffix(archive_path.file_name().unwrap())
+        .tempfile()?
+        .keep()
+        .unwrap();
+
+    new_f.write_all(
         desired.data(archive_map_data).map_err(|e| io::Error::new(io::ErrorKind::Other, e))?,
-        desired.offset().into(),
-    )))
+    )?;
+
+    Ok(Some(extracted_path))
 }
 
-pub fn try_extract_macho_fat_archive<'a>(
+pub fn try_extract_macho_fat_archive(
     sess: &Session,
-    archive_bytes: &'a [u8],
-) -> io::Result<Option<(&'a [u8], u64)>> {
+    archive_path: &Path,
+) -> io::Result<Option<PathBuf>> {
+    let archive_map = unsafe { Mmap::map(File::open(&archive_path)?)? };
     let target_arch = match sess.target.arch.as_ref() {
         "aarch64" => object::Architecture::Aarch64,
         "x86_64" => object::Architecture::X86_64,
         _ => return Ok(None),
     };
 
-    match object::macho::FatHeader::parse(archive_bytes) {
+    match object::macho::FatHeader::parse(&*archive_map) {
         Ok(h) if h.magic.get(object::endian::BigEndian) == object::macho::FAT_MAGIC => {
-            let archs = object::macho::FatHeader::parse_arch32(archive_bytes);
-            try_filter_fat_archs(archs, target_arch, archive_bytes)
+            let archs = object::macho::FatHeader::parse_arch32(&*archive_map);
+            try_filter_fat_archs(archs, target_arch, archive_path, &*archive_map)
         }
         Ok(h) if h.magic.get(object::endian::BigEndian) == object::macho::FAT_MAGIC_64 => {
-            let archs = object::macho::FatHeader::parse_arch64(archive_bytes);
-            try_filter_fat_archs(archs, target_arch, archive_bytes)
+            let archs = object::macho::FatHeader::parse_arch64(&*archive_map);
+            try_filter_fat_archs(archs, target_arch, archive_path, &*archive_map)
         }
         // Not a FatHeader at all, just return None.
         _ => Ok(None),
@@ -164,24 +173,21 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
         archive_path: &Path,
         mut skip: Box<dyn FnMut(&str) -> bool + 'static>,
     ) -> io::Result<()> {
-        let archive_map = unsafe { Mmap::map(File::open(&archive_path)?)? };
+        let mut archive_path = archive_path.to_path_buf();
+        if self.sess.target.llvm_target.contains("-apple-macosx") {
+            if let Some(new_archive_path) =
+                try_extract_macho_fat_archive(&self.sess, &archive_path)?
+            {
+                archive_path = new_archive_path
+            }
+        }
+
         if self.src_archives.iter().any(|archive| archive.0 == archive_path) {
             return Ok(());
         }
 
-        let (archive_bytes, offset) = if self.sess.target.llvm_target.contains("-apple-macosx") {
-            if let Some((sub_archive, archive_offset)) =
-                try_extract_macho_fat_archive(&self.sess, &*archive_map)?
-            {
-                (sub_archive, Some(archive_offset))
-            } else {
-                (&*archive_map, None)
-            }
-        } else {
-            (&*archive_map, None)
-        };
-
-        let archive = ArchiveFile::parse(&*archive_bytes)
+        let archive_map = unsafe { Mmap::map(File::open(&archive_path)?)? };
+        let archive = ArchiveFile::parse(&*archive_map)
             .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
         let archive_index = self.src_archives.len();
 
@@ -190,13 +196,9 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
             let file_name = String::from_utf8(entry.name().to_vec())
                 .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
             if !skip(&file_name) {
-                let mut range = entry.file_range();
-                if let Some(offset) = offset {
-                    range.0 += offset;
-                }
                 self.entries.push((
                     file_name.into_bytes(),
-                    ArchiveEntry::FromArchive { archive_index, file_range: range },
+                    ArchiveEntry::FromArchive { archive_index, file_range: entry.file_range() },
                 ));
             }
         }

src/ci/github-actions/ci.yml

@@ -77,7 +77,7 @@ x--expand-yaml-anchors--remove:
     <<: *base-job
 
   - &job-macos-xl
-    os: macos-12-xl
+    os: macos-latest  # We don't have an XL builder for this
     <<: *base-job
 
   - &job-windows-xl

src/tools/clippy/clippy_lints/src/format_args.rs

@@ -125,7 +125,7 @@ declare_clippy_lint! {
     /// nothing will be suggested, e.g. `println!("{0}={1}", var, 1+2)`.
     #[clippy::version = "1.66.0"]
     pub UNINLINED_FORMAT_ARGS,
-    style,
+    pedantic,
     "using non-inlined variables in `format!` calls"
 }