Add makefile support (#82)

* Adding makefile support for building with mmtk

Author: udesou

Make.inc

@@ -833,35 +833,25 @@ JCFLAGS += -DGC_DEBUG_ENV
 endif
 
 ifneq (${MMTK_PLAN},None)
-ifeq (${MMTK_JULIA_DIR},)
-$(error MMTK_JULIA_DIR must be set to use MMTk)
-endif
 JCXXFLAGS += -DMMTK_GC
 JCFLAGS += -DMMTK_GC
-ifeq (${MMTK_BUILD},)
-ifeq (debug,$(findstring debug,$(MAKECMDGOALS)))
-MMTK_BUILD = debug
-else
-MMTK_BUILD = release
-endif
-endif
+# Do a release build on the binding by default
+MMTK_BUILD ?= release
 ifeq (${MMTK_PLAN},Immix)
 JCXXFLAGS += -DMMTK_PLAN_IMMIX
 JCFLAGS += -DMMTK_PLAN_IMMIX
 else
 $(error "Unsupported MMTk plan: $(MMTK_PLAN)")
 endif
+
+# Location of mmtk-julia binding
+# (needed for api/*.h and .so file)
+MMTK_JULIA_DIR ?= $(BUILDROOT)/usr/lib/mmtk_julia
+
 MMTK_DIR = ${MMTK_JULIA_DIR}/mmtk
-MMTK_API_INC = $(MMTK_DIR)/api
-ifeq ($(OS),Linux)
-MMTK_LIB_NAME := libmmtk_julia.so
-else
-$(error "Unsupported OS for MMTk")
-endif
-MMTK_LIB_SRC := $(MMTK_DIR)/target/$(MMTK_BUILD)/$(MMTK_LIB_NAME)
-MMTK_LIB_DST := $(BUILDROOT)/usr/lib/$(MMTK_LIB_NAME)
+MMTK_API_INC = ${MMTK_DIR}/api
+
 MMTK_LIB := -lmmtk_julia
-LDFLAGS += -Wl,-rpath=$(MMTK_DIR)/target/$(MMTK_BUILD)/
 else
 MMTK_JULIA_INC :=
 MMTK_LIB :=
@@ -967,6 +957,21 @@ ARCH := $(BUILD_OS)
 endif
 endif
 
+# MMTk is only available on x86_64 Linux for now
+ifneq (${MMTK_PLAN},None)
+
+ifeq ($(OS),Linux)
+MMTK_LIB_NAME := libmmtk_julia.so
+else
+$(error "Unsupported OS for MMTk")
+endif
+
+ifneq ($(ARCH),x86_64)
+$(error "Unsupported build architecture for MMTk")
+endif
+
+endif
+
 # Detect common pre-SSE2 JULIA_CPU_TARGET values known not to work (#7185)
 ifeq ($(MARCH),)
 ifneq ($(findstring $(ARCH),i386 i486 i586 i686),)
@@ -1861,9 +1866,6 @@ PRINT_PERL = printf '    %b %b\n' $(PERLCOLOR)PERL$(ENDCOLOR) $(BINCOLOR)$(GOAL)
 PRINT_FLISP = printf '    %b %b\n' $(FLISPCOLOR)FLISP$(ENDCOLOR) $(BINCOLOR)$(GOAL)$(ENDCOLOR); $(1)
 PRINT_JULIA = printf '    %b %b\n' $(JULIACOLOR)JULIA$(ENDCOLOR) $(BINCOLOR)$(GOAL)$(ENDCOLOR); $(1)
 PRINT_DTRACE = printf '    %b %b\n' $(DTRACECOLOR)DTRACE$(ENDCOLOR) $(BINCOLOR)$(GOAL)$(ENDCOLOR); $(1)
-ifneq (${MMTK_PLAN},None)
-PRINT_MMTK = printf '    %b %b\n' $(LINKCOLOR)MMTK$(ENDCOLOR) $(BINCOLOR)$(GOAL)$(ENDCOLOR); $(1)
-endif
 
 else
 QUIET_MAKE =
@@ -1874,9 +1876,6 @@ PRINT_PERL = echo '$(subst ','\'',$(1))'; $(1)
 PRINT_FLISP = echo '$(subst ','\'',$(1))'; $(1)
 PRINT_JULIA = echo '$(subst ','\'',$(1))'; $(1)
 PRINT_DTRACE = echo '$(subst ','\'',$(1))'; $(1)
-ifneq (${MMTK_PLAN},None)
-PRINT_MMTK = echo '$(subst ','\'',$(1))'; $(1)
-endif
 
 endif # VERBOSE
 

deps/Makefile

@@ -26,7 +26,7 @@ BUILDDIR := $(BUILDDIR)$(MAYBE_HOST)
 # custom Makefile rules: openlibm dsfmt libsuitesparse lapack blastrampoline openblas utf8proc objconv libwhich
 # CMake libs: llvm llvmunwind libgit2 libssh2 mbedtls libtracyclient
 #
-# downloadable via git: llvm-svn, libuv, libopenlibm, utf8proc, libgit2, libssh2, libtracyclient
+# downloadable via git: llvm-svn, libuv, libopenlibm, utf8proc, libgit2, libssh2, libtracyclient, mmtk_julia
 #
 # to debug 'define' rules, replace eval at the usage site with info or error
 
@@ -195,14 +195,18 @@ DEP_LIBS += libwhich
 endif
 endif
 
+ifneq (${MMTK_PLAN},None)
+DEP_LIBS += mmtk_julia
+endif
+
 DEP_LIBS_STAGED := $(DEP_LIBS)
 
 # list all targets
 DEP_LIBS_STAGED_ALL := llvm llvm-tools clang llvmunwind unwind libuv pcre \
 	openlibm dsfmt blastrampoline openblas lapack gmp mpfr patchelf utf8proc \
 	objconv mbedtls libssh2 nghttp2 curl libgit2 libwhich zlib p7zip csl \
 	sanitizers libsuitesparse lld libtracyclient ittapi nvtx JuliaSyntax \
-	terminfo
+	terminfo mmtk_julia
 DEP_LIBS_ALL := $(DEP_LIBS_STAGED_ALL)
 
 ifneq ($(USE_BINARYBUILDER_OPENBLAS),0)
@@ -266,6 +270,9 @@ include $(SRCDIR)/p7zip.mk
 include $(SRCDIR)/libtracyclient.mk
 include $(SRCDIR)/terminfo.mk
 
+# MMTk
+include $(SRCDIR)/mmtk_julia.mk
+
 # vendored Julia libs
 include $(SRCDIR)/JuliaSyntax.mk
 

deps/checksums/mmtk_julia-05fb25cb03dfdf082e5023dd11bbcbb23a2b75d8.tar.gz/md5

@@ -0,0 +1 @@
+4b2bc06421faf022ee6bda09a7c87f7a

deps/checksums/mmtk_julia-05fb25cb03dfdf082e5023dd11bbcbb23a2b75d8.tar.gz/sha512

@@ -0,0 +1 @@
+1d9d500239fa9f3eb60404458fa5fe4e31d0dd4defbbf32ce84769b33a6c21e676a48be4f3b6dd3e114f367a2bd278c5ff03bb25f0617fa0101f470701fd2b8a

deps/mmtk_julia.mk

@@ -0,0 +1,71 @@
+## MMTK ##
+
+# Both MMTK_MOVING and MMTK_PLAN should be specified in the Make.user file.
+# At this point, since we only support non-moving this is always set to 0
+# FIXME: change it to `?:` when introducing moving plans
+MMTK_MOVING := 0
+MMTK_VARS := MMTK_PLAN=$(MMTK_PLAN) MMTK_MOVING=$(MMTK_MOVING)
+
+# Download the binding, build it from source
+ifeq (${MMTK_JULIA_DIR},$(BUILDROOT)/usr/lib/mmtk_julia)
+$(eval $(call git-external,mmtk_julia,MMTK_JULIA,,,$(BUILDDIR)))
+
+MMTK_JULIA_DIR=$(BUILDROOT)/deps/$(BUILDDIR)/$(MMTK_JULIA_SRC_DIR)
+MMTK_JULIA_LIB_PATH=$(MMTK_JULIA_DIR)/mmtk/target/$(MMTK_BUILD)
+PROJECT_DIRS := JULIA_PATH=$(JULIAHOME) JULIA_BUILDROOT=$(BUILDROOT) MMTK_JULIA_DIR=$(MMTK_JULIA_DIR)
+
+$(BUILDDIR)/$(MMTK_JULIA_SRC_DIR)/build-compiled: $(BUILDROOT)/usr/lib/libmmtk_julia.so
+	@echo 1 > $@
+
+# NB: use the absolute dir when creating the symlink
+$(BUILDROOT)/usr/lib/libmmtk_julia.so: $(MMTK_JULIA_LIB_PATH)/libmmtk_julia.so
+	@ln -sf $(MMTK_JULIA_LIB_PATH)/libmmtk_julia.so $@
+
+$(MMTK_JULIA_LIB_PATH)/libmmtk_julia.so: $(BUILDDIR)/$(MMTK_JULIA_SRC_DIR)/source-extracted
+	@$(PROJECT_DIRS) $(MMTK_VARS) $(MAKE) -C $(MMTK_JULIA_DIR) $(MMTK_BUILD)
+
+get-mmtk_julia: $(MMTK_JULIA_SRC_FILE)
+extract-mmtk_julia: $(BUILDDIR)/$(MMTK_JULIA_SRC_DIR)/source-extracted
+configure-mmtk_julia: extract-mmtk_julia
+compile-mmtk_julia: $(BUILDROOT)/usr/lib/libmmtk_julia.so
+fastcheck-mmtk_julia: #none
+check-mmtk_julia: compile-mmtk_julia
+
+$(eval $(call symlink_install,mmtk_julia,$$(MMTK_JULIA_SRC_DIR),$$(BUILDROOT)/usr/lib))
+
+# In this case, there is a custom version of the binding in MMTK_JULIA_DIR
+# Build it and symlink libmmtk_julia.so file into $(BUILDROOT)/usr/lib
+else
+
+PROJECT_DIRS := JULIA_PATH=$(JULIAHOME) JULIA_BUILDROOT=$(BUILDROOT) MMTK_JULIA_DIR=$(MMTK_JULIA_DIR)
+MMTK_JULIA_LIB_PATH=$(MMTK_JULIA_DIR)/mmtk/target/$(MMTK_BUILD)
+
+install-mmtk_julia: compile-mmtk_julia $(build_prefix)/manifest/mmtk_julia
+
+compile-mmtk_julia: $(BUILDROOT)/usr/lib/libmmtk_julia.so
+
+version-check-mmtk_julia: $(MMTK_JULIA_DIR)/mmtk/target/$(MMTK_BUILD)/libmmtk_julia.so
+
+# NB: This will NOT run `cargo build` if there are changes in the Rust source files
+# inside the binding repo. However the target below should remake the symlink if there
+# are changes in the libmmtk_julia.so from the custom MMTK_JULIA_DIR folder
+$(BUILDROOT)/usr/lib/libmmtk_julia.so: $(MMTK_JULIA_DIR)/mmtk/target/$(MMTK_BUILD)/libmmtk_julia.so
+	@ln -sf $(MMTK_JULIA_DIR)/mmtk/target/$(MMTK_BUILD)/libmmtk_julia.so $@
+
+$(MMTK_JULIA_DIR)/mmtk/target/$(MMTK_BUILD)/libmmtk_julia.so:
+	@$(PROJECT_DIRS) $(MMTK_VARS) $(MAKE) -C $(MMTK_JULIA_DIR) $(MMTK_BUILD)
+
+MMTK_JULIA_VER := mmtk_julia_custom
+
+UNINSTALL_mmtk_julia := $(MMTK_JULIA_VER) manual_mmtk_julia
+
+define manual_mmtk_julia
+uninstall-mmtk_julia:
+	-rm -f $(build_prefix)/manifest/mmtk_julia
+	-rm -f $(BUILDROOT)/usr/lib/libmmtk_julia.so
+endef
+
+$(build_prefix)/manifest/mmtk_julia: $(BUILDROOT)/usr/lib/libmmtk_julia.so
+	@echo $(UNINSTALL_mmtk_julia) > $@
+
+endif # MMTK_JULIA_DIR

deps/mmtk_julia.version

@@ -0,0 +1,4 @@
+MMTK_JULIA_BRANCH = master
+MMTK_JULIA_SHA1 = 05fb25cb03dfdf082e5023dd11bbcbb23a2b75d8
+MMTK_JULIA_GIT_URL := https://github.com/mmtk/mmtk-julia.git
+MMTK_JULIA_TAR_URL = https://github.com/mmtk/mmtk-julia/archive/refs/tags/v0.29.0.tar.gz

src/Makefile

@@ -29,7 +29,7 @@ ifeq ($(USECLANG),1)
 FLAGS += -Wno-return-type-c-linkage -Wno-atomic-alignment
 endif
 
-ifeq ($(WITH_MMTK), 1)
+ifneq (${MMTK_PLAN},None)
 FLAGS += -I$(MMTK_API_INC)
 endif
 
@@ -249,12 +249,6 @@ $(BUILDDIR)/%.h.gen : $(SRCDIR)/%.d
 	sed 's/JULIA_/JL_PROBE_/' $@ > $@.tmp
 	mv $@.tmp $@
 
-# Compile files from the binding side and copy so file into lib folder
-ifneq (${MMTK_PLAN},None)
-$(MMTK_LIB_DST): $(MMTK_LIB_SRC)
-	@$(call PRINT_MMTK, cp $< $@)
-endif
-
 $(BUILDDIR)/jl_internal_funcs.inc: $(SRCDIR)/jl_exported_funcs.inc
 	# Generate `.inc` file that contains a list of `#define` macros to rename functions defined in `libjulia-internal`
 	# to have a `ijl_` prefix instead of `jl_`, to denote that they are coming from `libjulia-internal`.  This avoids
@@ -420,13 +414,13 @@ $(BUILDDIR)/julia.expmap: $(SRCDIR)/julia.expmap.in $(JULIAHOME)/VERSION $(LLVM_
 	sed <'$<' >'$@' -e "s/@JULIA_SHLIB_SYMBOL_VERSION@/JL_LIBJULIA_$(SOMAJOR)/" \
 		        -e "s/@LLVM_SHLIB_SYMBOL_VERSION@/$(LLVM_SHLIB_SYMBOL_VERSION)/"
 
-$(build_shlibdir)/libjulia-internal.$(JL_MAJOR_MINOR_SHLIB_EXT): $(BUILDDIR)/julia.expmap $(OBJS) $(MMTK_LIB_DST) $(BUILDDIR)/flisp/libflisp.a $(BUILDDIR)/support/libsupport.a $(LIBUV)
+$(build_shlibdir)/libjulia-internal.$(JL_MAJOR_MINOR_SHLIB_EXT): $(BUILDDIR)/julia.expmap $(OBJS) $(BUILDDIR)/flisp/libflisp.a $(BUILDDIR)/support/libsupport.a $(LIBUV)
 	@$(call PRINT_LINK, $(CXXLD) $(call IMPLIB_FLAGS,$@) $(JCXXFLAGS) $(JL_CXXFLAGS) $(CXXLDFLAGS) $(SHIPFLAGS) $(OBJS) $(RPATH_LIB) -o $@ \
 		$(JLDFLAGS) $(BOLT_LDFLAGS) $(JLIBLDFLAGS) $(RT_RELEASE_LIBS) $(call SONAME_FLAGS,libjulia-internal.$(JL_MAJOR_SHLIB_EXT)))
 	@$(INSTALL_NAME_CMD)libjulia-internal.$(SHLIB_EXT) $@
 	$(DSYMUTIL) $@
 
-$(build_shlibdir)/libjulia-internal-debug.$(JL_MAJOR_MINOR_SHLIB_EXT): $(BUILDDIR)/julia.expmap $(DOBJS) $(MMTK_LIB_DST) $(BUILDDIR)/flisp/libflisp-debug.a $(BUILDDIR)/support/libsupport-debug.a $(LIBUV)
+$(build_shlibdir)/libjulia-internal-debug.$(JL_MAJOR_MINOR_SHLIB_EXT): $(BUILDDIR)/julia.expmap $(DOBJS) $(BUILDDIR)/flisp/libflisp-debug.a $(BUILDDIR)/support/libsupport-debug.a $(LIBUV)
 	@$(call PRINT_LINK, $(CXXLD) $(call IMPLIB_FLAGS,$@) $(JCXXFLAGS) $(JL_CXXFLAGS) $(CXXLDFLAGS) $(DEBUGFLAGS) $(DOBJS) $(RPATH_LIB) -o $@ \
 		$(JLDFLAGS) $(JLIBLDFLAGS) $(RT_DEBUG_LIBS) $(call SONAME_FLAGS,libjulia-internal-debug.$(JL_MAJOR_SHLIB_EXT)))
 	@$(INSTALL_NAME_CMD)libjulia-internal-debug.$(SHLIB_EXT) $@

src/gc-mmtk.c

@@ -162,7 +162,7 @@ JL_DLLEXPORT void jl_gc_set_max_memory(uint64_t max_mem) {
     // MMTk currently does not allow setting the heap size at runtime
 }
 
-inline void maybe_collect(jl_ptls_t ptls)
+STATIC_INLINE void maybe_collect(jl_ptls_t ptls)
 {
     // Just do a safe point for general maybe_collect
     jl_gc_safepoint_(ptls);
@@ -792,12 +792,12 @@ int jl_gc_classify_pools(size_t sz, int *osize)
 
 #define MMTK_MIN_ALIGNMENT 4
 // MMTk assumes allocation size is aligned to min alignment.
-inline size_t mmtk_align_alloc_sz(size_t sz) JL_NOTSAFEPOINT
+STATIC_INLINE size_t mmtk_align_alloc_sz(size_t sz) JL_NOTSAFEPOINT
 {
     return (sz + MMTK_MIN_ALIGNMENT - 1) & ~(MMTK_MIN_ALIGNMENT - 1);
 }
 
-inline void* bump_alloc_fast(MMTkMutatorContext* mutator, uintptr_t* cursor, uintptr_t limit, size_t size, size_t align, size_t offset, int allocator) {
+STATIC_INLINE void* bump_alloc_fast(MMTkMutatorContext* mutator, uintptr_t* cursor, uintptr_t limit, size_t size, size_t align, size_t offset, int allocator) {
     intptr_t delta = (-offset - *cursor) & (align - 1);
     uintptr_t result = *cursor + (uintptr_t)delta;
 
@@ -809,7 +809,7 @@ inline void* bump_alloc_fast(MMTkMutatorContext* mutator, uintptr_t* cursor, uin
     }
 }
 
-inline void* mmtk_immix_alloc_fast(MMTkMutatorContext* mutator, size_t size, size_t align, size_t offset) {
+STATIC_INLINE void* mmtk_immix_alloc_fast(MMTkMutatorContext* mutator, size_t size, size_t align, size_t offset) {
     ImmixAllocator* allocator = &mutator->allocators.immix[MMTK_DEFAULT_IMMIX_ALLOCATOR];
     return bump_alloc_fast(mutator, (uintptr_t*)&allocator->cursor, (intptr_t)allocator->limit, size, align, offset, 0);
 }
@@ -818,17 +818,17 @@ inline void mmtk_immix_post_alloc_slow(MMTkMutatorContext* mutator, void* obj, s
     mmtk_post_alloc(mutator, obj, size, 0);
 }
 
-inline void mmtk_immix_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) {
+STATIC_INLINE void mmtk_immix_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) {
     // FIXME: for now, we do nothing
     // but when supporting moving, this is where we set the valid object (VO) bit
 }
 
-inline void* mmtk_immortal_alloc_fast(MMTkMutatorContext* mutator, size_t size, size_t align, size_t offset) {
+STATIC_INLINE void* mmtk_immortal_alloc_fast(MMTkMutatorContext* mutator, size_t size, size_t align, size_t offset) {
     BumpAllocator* allocator = &mutator->allocators.bump_pointer[MMTK_IMMORTAL_BUMP_ALLOCATOR];
     return bump_alloc_fast(mutator, (uintptr_t*)&allocator->cursor, (uintptr_t)allocator->limit, size, align, offset, 1);
 }
 
-inline void mmtk_immortal_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) {
+STATIC_INLINE void mmtk_immortal_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) {
     // FIXME: Similarly, for now, we do nothing
     // but when supporting moving, this is where we set the valid object (VO) bit
     // and log (old gen) bit