Vendor the terminfo database for use with base/terminfo.jl (#55411)

This adds the `terminfo` database to `deps/`, providing a better user
experience on systems that don't have `terminfo` on the system by
default. The database is built using BinaryBuilder but is not actually
platform-specific (it's built for `AnyPlatform`) and as such, this
fetches the artifact directly rather than adding a new JLL to stdlib,
and it requires no compilation.

A build flag, `WITH_TERMINFO`, is added here and assumed true by
default, allowing users to set `WITH_TERMINFO=0` in Make.user to avoid
bundling `terminfo` should they want to do so.

The lookup policy for `terminfo` entries is still compliant with what's
described in `terminfo(5)`; the bundled directory is taken to be the
first "compiled in" location, i.e. prepended to `@TERMINFO_DIRS@`. This
allows any user settings that exist locally, such as custom entries or
locations, to take precedence.

Fixes #55274

Co-authored-by: Mosè Giordano <[email protected]>

Author: ararslan

Makefile

@@ -403,6 +403,10 @@ endif
 	# Install appdata file
 	mkdir -p $(DESTDIR)$(datarootdir)/metainfo/
 	$(INSTALL_F) $(JULIAHOME)/contrib/julia.appdata.xml $(DESTDIR)$(datarootdir)/metainfo/
+	# Install terminal info database
+ifneq ($(WITH_TERMINFO),0)
+	cp -R -L $(build_datarootdir)/terminfo $(DESTDIR)$(datarootdir)
+endif
 
 	# Update RPATH entries and JL_SYSTEM_IMAGE_PATH if $(private_libdir_rel) != $(build_private_libdir_rel)
 ifneq ($(private_libdir_rel),$(build_private_libdir_rel))

NEWS.md

@@ -171,6 +171,10 @@ Deprecated or removed
 External dependencies
 ---------------------
 
+- The terminal info database, `terminfo`, is now vendored by default, providing a better
+  REPL user experience when `terminfo` is not available on the system. Julia can be built
+  without vendoring the database using the Makefile option `WITH_TERMINFO=0`. ([#55411])
+
 Tooling Improvements
 --------------------
 

base/terminfo.jl

@@ -245,7 +245,8 @@ end
 Locate the terminfo file for `term`, return `nothing` if none could be found.
 
 The lookup policy is described in `terminfo(5)` "Fetching Compiled
-Descriptions".
+Descriptions". A terminfo database is included by default with Julia and is
+taken to be the first entry of `@TERMINFO_DIRS@`.
 """
 function find_terminfo_file(term::String)
     isempty(term) && return
@@ -261,15 +262,23 @@ function find_terminfo_file(term::String)
         append!(terminfo_dirs,
                 replace(split(ENV["TERMINFO_DIRS"], ':'),
                         "" => "/usr/share/terminfo"))
+    push!(terminfo_dirs, normpath(Sys.BINDIR, DATAROOTDIR, "terminfo"))
     Sys.isunix() &&
         push!(terminfo_dirs, "/etc/terminfo", "/lib/terminfo", "/usr/share/terminfo")
     for dir in terminfo_dirs
         if isfile(joinpath(dir, chr, term))
             return joinpath(dir, chr, term)
         elseif isfile(joinpath(dir, chrcode, term))
             return joinpath(dir, chrcode, term)
+        elseif isfile(joinpath(dir, lowercase(chr), lowercase(term)))
+            # The vendored terminfo database is fully lowercase to avoid issues on
+            # case-sensitive filesystems. On Unix-like systems, terminfo files with
+            # different cases are hard links to one another, so this is still
+            # correct for non-vendored terminfo, just redundant.
+            return joinpath(dir, lowercase(chr), lowercase(term))
         end
     end
+    return nothing
 end
 
 """

deps/Makefile

@@ -175,6 +175,10 @@ ifeq ($(WITH_NVTX),1)
 DEP_LIBS += nvtx
 endif
 
+ifneq ($(WITH_TERMINFO),0)
+DEP_LIBS += terminfo
+endif
+
 # Only compile standalone LAPACK if we are not using OpenBLAS.
 # OpenBLAS otherwise compiles LAPACK as part of its build.
 # This is useful where one wants to use the vendor BLAS, but
@@ -197,7 +201,8 @@ DEP_LIBS_STAGED := $(DEP_LIBS)
 DEP_LIBS_STAGED_ALL := llvm llvm-tools clang llvmunwind unwind libuv pcre \
 	openlibm dsfmt blastrampoline openblas lapack gmp mpfr patchelf utf8proc \
 	objconv mbedtls libssh2 nghttp2 curl libgit2 libwhich zlib p7zip csl \
-	sanitizers libsuitesparse lld libtracyclient ittapi nvtx JuliaSyntax
+	sanitizers libsuitesparse lld libtracyclient ittapi nvtx JuliaSyntax \
+	terminfo
 DEP_LIBS_ALL := $(DEP_LIBS_STAGED_ALL)
 
 ifneq ($(USE_BINARYBUILDER_OPENBLAS),0)
@@ -259,6 +264,7 @@ include $(SRCDIR)/libgit2.mk
 include $(SRCDIR)/libwhich.mk
 include $(SRCDIR)/p7zip.mk
 include $(SRCDIR)/libtracyclient.mk
+include $(SRCDIR)/terminfo.mk
 
 # vendored Julia libs
 include $(SRCDIR)/JuliaSyntax.mk

deps/checksums/terminfo

@@ -0,0 +1,2 @@
+TermInfoDB-v2023.12.9.any.tar.gz/md5/573d9b5adaf6af500e3dfae6e3d15ebf
+TermInfoDB-v2023.12.9.any.tar.gz/sha512/e0a5bfe54346f9d5690a840628b329f6fac7375b0d29337bc70813ae3553a72bb397f8034d221c544289e40c4cfc685d5805777b7528f05bbe0123b5905c24a4

deps/terminfo.mk

@@ -0,0 +1,43 @@
+## TERMINFO-DB ##
+include $(SRCDIR)/terminfo.version
+
+$(SRCCACHE)/TermInfoDB-v$(TERMINFO_VER).any.tar.gz: | $(SRCCACHE)
+	$(JLDOWNLOAD) $@ https://github.com/JuliaBinaryWrappers/TermInfoDB_jll.jl/releases/download/$(TERMINFO_TAG)/TermInfoDB.v$(TERMINFO_VER).any.tar.gz
+	touch -c $@
+
+$(BUILDDIR)/TermInfoDB-v$(TERMINFO_VER)/source-extracted: $(SRCCACHE)/TermInfoDB-v$(TERMINFO_VER).any.tar.gz
+	$(JLCHECKSUM) $<
+	rm -rf $(dir $@)
+	mkdir -p $(dir $@)
+	$(TAR) -C $(dir $@) --strip-components 1 -xf $<
+	echo 1 > $@
+
+checksum-terminfo: $(SRCCACHE)/TermInfoDB-v$(TERMINFO_VER).any.tar.gz
+	$(JLCHECKSUM) $<
+
+$(BUILDDIR)/TermInfoDB-v$(TERMINFO_VER)/build-compiled: $(BUILDDIR)/TermInfoDB-v$(TERMINFO_VER)/source-extracted
+	echo 1 > $@
+
+$(BUILDDIR)/TermInfoDB-v$(TERMINFO_VER)/build-checked: $(BUILDDIR)/TermInfoDB-v$(TERMINFO_VER)/build-compiled
+	echo 1 > $@
+
+define TERMINFO_INSTALL
+	mkdir -p $2/$$(build_datarootdir)
+	cp -R $1/terminfo $2/$$(build_datarootdir)
+endef
+$(eval $(call staged-install, \
+	terminfo,TermInfoDB-v$(TERMINFO_VER), \
+	TERMINFO_INSTALL,,,,))
+
+clean-terminfo:
+	-rm -f $(BUILDDIR)/TermInfoDB-v$(TERMINFO_VER)/build-compiled
+
+distclean-terminfo:
+	rm -rf $(SRCCACHE)/TermInfoDB*.tar.gz $(SRCCACHE)/TermInfoDB-v$(TERMINFO_VER) $(BUILDDIR)/TermInfoDB-v$(TERMINFO_VER)
+
+get-terminfo: $(SRCCACHE)/TermInfoDB-v$(TERMINFO_VER).any.tar.gz
+extract-terminfo: $(BUILDDIR)/TermInfoDB-v$(TERMINFO_VER)/source-extracted
+configure-terminfo: extract-terminfo
+compile-terminfo: $(BUILDDIR)/TermInfoDB-v$(TERMINFO_VER)/build-compiled
+fastcheck-terminfo: check-terminfo
+check-terminfo: $(BUILDDIR)/TermInfoDB-v$(TERMINFO_VER)/build-checked

deps/terminfo.version

@@ -0,0 +1,3 @@
+# -*- makefile -*-
+TERMINFO_VER := 2023.12.9
+TERMINFO_TAG := TermInfoDB-v$(TERMINFO_VER)+0

stdlib/REPL/test/precompilation.jl

@@ -15,8 +15,11 @@ if !Sys.iswindows()
     @testset "No interactive startup compilation" begin
         f, _ = mktemp()
 
-        # start an interactive session
-        cmd = `$(Base.julia_cmd()[1]) --trace-compile=$f -q --startup-file=no -i`
+        # start an interactive session, ensuring `TERM` is unset since it can trigger
+        # different amounts of precompilation stemming from `base/terminfo.jl` depending
+        # on the value, making the test here unreliable
+        cmd = addenv(`$(Base.julia_cmd()[1]) --trace-compile=$f -q --startup-file=no -i`,
+                     Dict("TERM" => ""))
         pts, ptm = open_fake_pty()
         p = run(cmd, pts, pts, pts; wait=false)
         Base.close_stdio(pts)