Devtools darwin (spack#46910)

* stacks: add a stack for devtools on darwin

After getting this whole mess building on darwin, let's keep it that
way, and maybe make it so we have some non-ML darwin binaries in spack
as well.

* reuse: false for devtools

* dtc: fix darwin dylib name and id

On mac the convention is `lib<name>.<num>.dylib`, while the makefile
creates a num suffixed one by default. The id in the file is also a
local name rather than rewritten to the full path, this fixes both
problems.

* node-js: make whereis more deterministic

* relocation(darwin): catch Mach-O load failure

The MachO library can throw an exception rather than return no headers,
this happened in an elf file in the test data of go-bootstrap.  Trying
catching the exception and moving on for now.  May also need to look
into why we're trying to rewrite an elf file.

* qemu: add darwin flags to clear out warnings

There's a build failure for qemu in CI, but it's invisible because of
the immense mass of warning output.  Explicitly specify the target macos
version and remove the extraneous unknown-warning-option flag.

* dtc: libyaml is also a link dependency

libyaml is required at runtime to run the dtc binary, lack of it caused
the ci for qemu to fail when the library wasn't found.

lib/spack/spack/relocate.py

@@ -283,7 +283,11 @@ def modify_macho_object(cur_path, rpaths, deps, idpath, paths_to_paths):
 
 def macholib_get_paths(cur_path):
     """Get rpaths, dependent libraries, and library id of mach-o objects."""
-    headers = macholib.MachO.MachO(cur_path).headers
+    headers = []
+    try:
+        headers = macholib.MachO.MachO(cur_path).headers
+    except ValueError:
+        pass
     if not headers:
         tty.warn("Failed to read Mach-O headers: {0}".format(cur_path))
         commands = []

share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml

@@ -508,6 +508,30 @@ developer-tools-manylinux2014-build:
     - artifacts: True
       job: developer-tools-manylinux2014-generate
 
+###########################################
+# Build tests for different developer tools
+# darwin
+###########################################
+.developer-tools-darwin:
+  extends: [ ".darwin_aarch64" ]
+  variables:
+    SPACK_CI_STACK_NAME: developer-tools-darwin
+
+developer-tools-darwin-generate:
+  tags: [ "macos-ventura", "apple-clang-15", "aarch64-macos" ]
+  extends: [ ".developer-tools-darwin", ".generate-base"]
+
+developer-tools-darwin-build:
+  extends: [ ".developer-tools-darwin", ".build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: developer-tools-darwin-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: developer-tools-darwin-generate
+
 #########################################
 # RADIUSS
 #########################################

share/spack/gitlab/cloud_pipelines/stacks/developer-tools-darwin/spack.yaml

@@ -0,0 +1,75 @@
+spack:
+  view: false
+  packages:
+    all:
+      require:
+      - target=aarch64
+  concretizer:
+    unify: true
+    reuse: false
+  specs:
+      # editors
+    - neovim~no_luajit
+    - py-pynvim
+    - emacs+json~native+treesitter   # TODO native not supported until gcc builds on darwin
+      # - tree-sitter is a dep, should also have cli but no package
+    - nano   # just in case
+      # tags and scope search helpers
+    - universal-ctags   # only maintained ctags, works better with c++
+    - direnv
+      # runtimes and compilers
+    - python
+    - llvm+link_llvm_dylib+lld~lldb~polly+python build_type=MinSizeRel   # for clangd, clang-format
+    - node-js   # for editor plugins etc., pyright language server
+    - npm
+    - cmake
+    - libtool
+    - go   # to build fzf, gh, hub
+    - rust+dev   # fd, ripgrep, hyperfine, exa, rust-analyzer
+      # styling and lints
+    - astyle
+    - cppcheck
+    - uncrustify
+    - py-fprettify
+    - py-fortran-language-server
+    - py-python-lsp-server
+      # cli dev tools
+    - ripgrep
+    - gh
+    - fd
+    # - bfs # liburing: /usr/include/linux/ipv6.h:19:8: error: redefinition of 'struct in6_pktinfo'
+    - fzf
+    - tree
+    - jq
+    - py-yq
+    - hub
+    - ncdu
+    - eza
+    - lsd
+    - hyperfine
+    - htop
+    - tmux
+    - ccache
+      # ensure we can use a jobserver build and do this fast
+    - gmake
+    - ninja   # should be @kitware, can't be because of meson requirement
+    - libtree
+    - sed
+    - which
+    - flex
+    - graphviz
+    - doxygen
+    - meson
+    - lima
+
+  ci:
+    pipeline-gen:
+    - build-job-remove:
+        tags: [ spack, public ]
+    - build-job:
+        variables:
+          CI_GPG_KEY_ROOT: /etc/protected-runner
+        tags: [ "macos-ventura", "apple-clang-15", "aarch64-macos" ]
+
+  cdash:
+    build-group: Developer Tools Darwin

share/spack/gitlab/cloud_pipelines/stacks/developer-tools-manylinux2014/spack.yaml

@@ -5,6 +5,7 @@ spack:
       require: target=x86_64_v3
   concretizer:
     unify: true
+    reuse: false
   definitions:
   - default_specs:
       # editors

var/spack/repos/builtin/packages/dtc/package.py

@@ -24,9 +24,9 @@ class Dtc(MakefilePackage):
     # Build error with flex 2.6.3
     #   (convert-dtsv0-lexer.lex.c:398: error: "yywrap" redefined)
     depends_on("[email protected]:", type="build")
-    depends_on("libyaml", type="build")
     depends_on("pkgconfig", type="build")
     depends_on("python", type="build")
+    depends_on("libyaml", type=("build", "link"))
 
     def edit(self, spec, prefix):
         makefile = FileFilter("Makefile")
@@ -35,3 +35,14 @@ def edit(self, spec, prefix):
             makefile.filter(
                 r"WARNINGS = -Wall", "WARNINGS = -Wall -Wno-unused-command-line-argument"
             )
+
+        if self.spec.satisfies("platform=darwin"):
+            libfdt_makefile = FileFilter("libfdt/Makefile.libfdt")
+            libfdt_makefile.filter(
+                r"LIBFDT_soname = .*", "LIBFDT_soname = libfdt.1.$(SHAREDLIB_EXT)"
+            )
+
+    @run_after("install")
+    def darwin_fix(self):
+        if self.spec.satisfies("platform=darwin"):
+            fix_darwin_install_name(self.prefix.lib)

var/spack/repos/builtin/packages/node-js/package.py

@@ -113,8 +113,14 @@ def configure_args(self):
             #
             # /usr/bin/libtool
             # libtool: /usr/bin/libtool
+            #
+            # We specify -M -f (an empty list of man-path entries) to prevent man-page
+            # searching to avoid an Illegal seek error processing manpath results in CI,
+            # which prevents the last form:
             # libtool: /usr/bin/libtool /Applications/Xcode.app/.../share/man/man1/libtool.1
-            process_pipe = subprocess.Popen(["whereis", "libtool"], stdout=subprocess.PIPE)
+            process_pipe = subprocess.Popen(
+                ["whereis", "-M", "-f", "libtool"], stdout=subprocess.PIPE
+            )
             result_whereis_list = process_pipe.communicate()[0].strip().split()
             if len(result_whereis_list) == 1:
                 result_whereis = result_whereis_list[0]

var/spack/repos/builtin/packages/qemu/package.py

@@ -139,7 +139,7 @@ class Qemu(AutotoolsPackage):
 
     @when("@9:")
     def configure_args(self):
-        return [
+        args = [
             "--disable-bsd-user",
             "--disable-guest-agent",
             "--disable-sdl",
@@ -155,3 +155,11 @@ def configure_args(self):
             "--enable-zstd",
             "--disable-docs",
         ]
+        extra_cflags = "-Wno-unknown-warning-option"
+        if self.spec.satisfies("%apple-clang platform=darwin"):
+            # qemu 9: uses pthread_jit_write_protect_np which requires OSX 11.0 or newer
+            extra_cflags += " -mmacosx-version-min=11.0"
+        args.append(f"--extra-cflags={extra_cflags}")
+        args.append(f"--extra-cxxflags={extra_cflags}")
+
+        return args