Merge pull request #457 from osresearch/make-4.2.1

Make 4.2.1, hashes, external coreboot crossgcc, circleci and other fixes

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: ubuntu:16.04
+      - image: osresearch/heads-ubuntu:16.04
     steps:
       - run:
           name: Install dependencies
@@ -29,25 +29,70 @@ jobs:
               gnat \
               cpio \
               ccache \
+              lzma \
 
       - checkout
 
       - run:
           name: Bootstrap make
           command: |
-            make `pwd`/build/make-4.2/make
+            make -j4 bootstrap
+
       - run:
-          name: Bootstrap musl
+          name: Bootstrap coreboot-gcc
           command: |
-            ./build/make-4.2/make -j4 musl-cross V=1
+            ./build/make-4.2.1/make \
+                TOOLCHAIN=/home/builder/heads \
+                V=1 \
+                BOARD=qemu-coreboot \
+                coreboot-gcc
+
+
       - run:
           name: qemu-coreboot
           command: |
-            ./build/make-4.2/make -j4 BOARD=qemu-coreboot V=1
+            ./build/make-4.2.1/make \
+                TOOLCHAIN=/home/builder/heads \
+                V=1 \
+                -j4 \
+                BOARD=qemu-coreboot \
+
+      - store-artifacts:
+          path: build/qemu-coreboot/coreboot.rom
+      - store-artifacts:
+          path: build/qemu-coreboot/hashes.txt
+
+
       - run:
           name: qemu-linuxboot
           command: |
-            ./build/make-4.2/make -j4 BOARD=qemu-linuxboot V=1
+            ./build/make-4.2.1/make \
+                TOOLCHAIN=/home/builder/heads \
+                V=1 \
+                -j4 \
+                BOARD=qemu-linuxboot \
+
+      - store-artifacts:
+          path: build/qemu-linuxboot/linuxboot.rom
+      - store-artifacts:
+          path: build/qemu-linuxboot/hashes.txt
+
+
+      - run:
+          name: x230
+          command: |
+            ./build/make-4.2.1/make \
+                TOOLCHAIN=/home/builder/heads \
+                -j4 \
+                V=1 \
+                BOARD=x230 \
+
+      - store-artifacts:
+          path: build/x230/coreboot.rom
+      - store-artifacts:
+          path: build/x230/hashes.txt
+
+
 workflows:
   version: 2
   build_and_test:

Makefile

@@ -3,7 +3,7 @@ all:
 
 ifneq "$(TOOLCHAIN)" ""
 $(info Using $(TOOLCHAIN) for cross compiler and packages)
-packages	:= $(TOOLCHAIN)/packages
+packages	?= $(TOOLCHAIN)/packages
 CROSS		:= $(TOOLCHAIN)/crossgcc/x86_64-linux-musl/bin/x86_64-linux-musl-
 endif
 
@@ -22,21 +22,23 @@ CPUS		:= $(shell nproc)
 # Create the log directory if it doesn't already exist
 BUILD_LOG := $(shell mkdir -p "$(log_dir)" )
 
+WGET ?= wget
+
 # Check that we have a correct version of make
 # that matches at least the major version
 LOCAL_MAKE_VERSION := $(shell $(MAKE) --version | head -1 | cut -d' ' -f3)
 include modules/make
 
 ifneq "" "$(filter $(make_version)%,$(LOCAL_MAKE_VERSION))"
 
+# Timestamps should be in ISO format
+DATE=`date --rfc-3339=seconds`
+
 # This is the correct version of Make
 
 BOARD		?= qemu-coreboot
 CONFIG		:= $(pwd)/boards/$(BOARD)/$(BOARD).config
 
-# Create the board output directory if it doesn't already exist
-BOARD_LOG	:= $(shell mkdir -p "$(build)/$(BOARD)" )
-
 ifneq "y" "$(shell [ -r '$(CONFIG)' ] && echo y)"
 $(error $(CONFIG): board configuration does not exist)
 endif
@@ -56,8 +58,15 @@ GIT_STATUS	:= $(shell \
 		echo dirty ; \
 	fi)
 
-# Timestamps should be in ISO format
-DATE=`date --rfc-3339=seconds`
+# record the build date / git hashes and other files here
+HASHES		:= $(build)/$(BOARD)/hashes.txt
+
+# Create the board output directory if it doesn't already exist
+BOARD_LOG	:= $(shell \
+	mkdir -p "$(build)/$(BOARD)" ; \
+	echo "$(DATE) $(GIT_HASH) $(GIT_STATUS)" > "$(HASHES)" ; \
+)
+
 
 # If V is set in the environment, do not redirect the tee
 # command to /dev/null.
@@ -127,6 +136,9 @@ else
 $(error "$(BOARD): neither CONFIG_COREBOOT nor CONFIG_LINUXBOOT is set?")
 endif
 
+all:
+	@sha256sum $< | tee -a "$(HASHES)"
+
 # Disable all built in rules
 .INTERMEDIATE:
 .SUFFIXES:
@@ -180,6 +192,15 @@ define do-cpio =
 		echo "$(DATE) UNCHANGED $(1:$(pwd)/%=%)" ; \
 		rm "$1.tmp" ; \
 	fi
+	@sha256sum "$1" | tee -a "$(HASHES)"
+	$(call do,HASHES   , $1,\
+		( cd "$2"; \
+		echo "-----" ; \
+		find . -type f -print0 \
+		| xargs -0 sha256sum ; \
+		echo "-----" ; \
+		) >> "$(HASHES)" \
+	)
 endef
 
 define do-copy =
@@ -230,10 +251,10 @@ define define_module =
     # wget creates it early, so we have to cleanup if it fails
     $(packages)/$($1_tar):
 	$(call do,WGET,$($1_url),\
-		if ! wget -O "$$@" $($1_url) ; then \
-			rm -f "$$@" ; \
+		if ! $(WGET) -O "$$@.tmp" $($1_url) ; then \
 			exit 1 ; \
-		fi \
+		fi ; \
+		mv "$$@.tmp" "$$@" \
 	)
     $(packages)/.$1-$($1_version)_verify: $(packages)/$($1_tar)
 	echo "$($1_hash)  $$^" | sha256sum --check -
@@ -474,11 +495,11 @@ $(build)/$(initrd_dir)/initrd.cpio.xz: $(initrd-y)
 	)
 	@if ! cmp --quiet "$@.tmp" "$@" ; then \
 		mv "$@.tmp" "$@" ; \
-		sha256sum "$(@:$(pwd)/%=%)" ; \
 	else \
 		echo "$(DATE) UNCHANGED $(@:$(pwd)/%=%)" ; \
 		rm "$@.tmp" ; \
 	fi
+	@sha256sum "$(@:$(pwd)/%=%)" | tee -a "$(HASHES)"
 
 #
 # The heads.cpio is built from the initrd directory in the
@@ -557,30 +578,40 @@ $(eval $(shell echo >&2 "$(DATE) Wrong make detected: $(LOCAL_MAKE_VERSION)"))
 HEADS_MAKE := $(build)/$(make_dir)/make
 
 # Once we have a proper Make, we can just pass arguments into it
-all bootstrap linux cpio: $(HEADS_MAKE)
+all linux cpio run: $(HEADS_MAKE)
 	LANG=C MAKE=$(HEADS_MAKE) $(HEADS_MAKE) $(MAKE_JOBS) $@
-%.clean %.vol: $(HEADS_MAKE)
+%.clean %.vol %.menuconfig: $(HEADS_MAKE)
 	LANG=C MAKE=$(HEADS_MAKE) $(HEADS_MAKE) $@
 
+bootstrap: $(HEADS_MAKE)
+
 # How to download and build the correct version of make
-$(HEADS_MAKE): $(build)/$(make_dir)/Makefile
+$(packages)/$(make_tar):
+	$(WGET) -O "$@.tmp" "$(make_url)"
+	if ! echo "$(make_hash)  $@.tmp" | sha256sum --check -; then \
+		exit 1 ; \
+	fi
+	mv "$@.tmp" "$@"
+
+$(build)/$(make_dir)/.extract: $(packages)/$(make_tar)
+	tar xf "$<" -C "$(build)"
+	touch "$@"
+
+$(build)/$(make_dir)/.patch: patches/make-$(make_version).patch $(build)/$(make_dir)/.extract
+	( cd "$(dir $@)" ; patch -p1 ) < "$<"
+	touch "$@"
+
+$(build)/$(make_dir)/.configured: $(build)/$(make_dir)/.patch
+	cd "$(dir $@)" ; \
+	./configure 2>&1 \
+	| tee "$(log_dir)/make.configure.log" \
+	$(VERBOSE_REDIRECT)
+	touch "$@"
+
+$(HEADS_MAKE): $(build)/$(make_dir)/.configured
 	make -C "$(dir $@)" $(MAKE_JOBS) \
 		2>&1 \
 		| tee "$(log_dir)/make.log" \
 		$(VERBOSE_REDIRECT)
 
-$(build)/$(make_dir)/Makefile: $(packages)/$(make_tar)
-	tar xf "$<" -C build/
-	cd "$(dir $@)" ; ./configure \
-		2>&1 \
-		| tee "$(log_dir)/make.configure.log" \
-		$(VERBOSE_REDIRECT)
-
-$(packages)/$(make_tar):
-	wget -O "$@" "$(make_url)"
-	if ! echo "$(make_hash)  $@" | sha256sum --check -; then \
-		$(MV) "$@" "$@.failed"; \
-		false; \
-	fi
-
 endif

modules/coreboot

@@ -41,13 +41,33 @@ coreboot_target := \
 coreboot_output := coreboot.rom
 coreboot_depend += linux initrd
 
+COREBOOT_XGCC_REL := $(coreboot_base_dir)/util/crossgcc/xgcc
+COREBOOT_XGCC_PATH := $(build)/$(COREBOOT_XGCC_REL)
+COREBOOT_XGCC := $(COREBOOT_XGCC_PATH)/bin/i386-elf-gcc
+
 # hack to force a build dependency on the cross compiler
-$(build)/$(coreboot_dir)/.configured: $(build)/$(coreboot_base_dir)/util/crossgcc/xgcc/bin/i386-elf-gcc
-$(build)/$(coreboot_base_dir)/util/crossgcc/xgcc/bin/i386-elf-gcc: $(build)/$(coreboot_base_dir)/.canary
+coreboot-gcc $(build)/$(coreboot_dir)/.configured: $(COREBOOT_XGCC)
+
+ifeq "$(TOOLCHAIN)" ""
+# Force a rebuild of the entire coreboot toolchain
+$(COREBOOT_XGCC): $(build)/$(coreboot_base_dir)/.canary
 	echo '******* Building crossgcc-i386 (this might take a while) ******'
 	$(MAKE) -C "$(build)/$(coreboot_base_dir)" CPUS=`nproc` crossgcc-i386
 	#echo '******* Building crossgcc-arm (this might take a while) ******'
 	#$(MAKE) -C "$(build)/$(coreboot_base_dir)" crossgcc-arm
+else
+# Use the pre-build one from the external toolchain build
+$(COREBOOT_XGCC): $(build)/$(coreboot_base_dir)/.canary
+	if [ ! -e "$(TOOLCHAIN)/build/$(COREBOOT_XGCC_REL)" ]; then \
+		echo >&2 "ERROR: TOOLCHAIN=$(TOOLCHAIN) does not have coreboot" ; \
+		exit 1 ; \
+	fi
+	if [ ! -e "$(COREBOOT_XGCC_PATH)" ]; then \
+		ln -s \
+			"$(TOOLCHAIN)/build/$(COREBOOT_XGCC_REL)" \
+			"$(COREBOOT_XGCC_PATH)" ; \
+	fi
+endif
 
 # Force a rebuild if the inputs have changed
 $(build)/$(coreboot_dir)/.build: \

modules/linux

@@ -34,6 +34,16 @@ linux_configure := \
 
 linux_output += arch/x86/boot/bzImage
 
+# Once we have extracted the kernel tar file, install the headers
+# so that other submodules can make use of them.
+$(INSTALL)/include/linux/limits.h: $(build)/$(linux_base_dir)/.canary
+	$(MAKE) \
+		-C "$(build)/$(linux_base_dir)" \
+		INSTALL_HDR_PATH="$(INSTALL)" \
+		O="$(linux_dir)" \
+		KCONFIG_CONFIG="$(pwd)/$(linux_kconfig)" \
+		headers_install
+
 
 # qemu
 linux_modules-$(CONFIG_LINUX_E1000)	+= drivers/net/ethernet/intel/e1000/e1000.ko
@@ -145,6 +155,7 @@ $(build)/$(BOARD)/modules.cpio: $(build)/$(linux_dir)/.build
 $(build)/$(BOARD)/bzImage: $(build)/$(linux_dir)/.build
 	$(call do-copy,$(dir $<)/$(linux_output),$@)
 	@touch $@ # force a timestamp update
+	@sha256sum "$@" | tee -a "$(HASHES)"
 
 
 # menuconfig target allows us to easily reconfigure this Linux kernel

modules/make

@@ -3,11 +3,12 @@
 # target platform.
 #modules += make
 
-make_version := 4.2
+make_version := 4.2.1
 make_dir := make-$(make_version)
 make_tar := make-$(make_version).tar.bz2
-make_url := https://ftpmirror.gnu.org/make/$(make_tar)
-make_hash := 4e5ce3b62fe5d75ff8db92b7f6df91e476d10c3aceebf1639796dc5bfece655f
+
+make_url := http://gnu.mirror.constant.com/make/$(make_tar)
+make_hash := d6e262bf3601b42d2b1e4ef8310029e1dcf20083c5446b4b7aa67081fdffc589
 
 # This is built for the local machine, not the target, so it doesn't have any
 # of the build instructions.

modules/musl

@@ -39,12 +39,3 @@ musl_depends := musl-cross
 # Fake a target so that musl will force a header install by the
 # Linux kernel sources.
 $(build)/$(musl_dir)/.build: $(INSTALL)/include/linux/limits.h
-
-$(INSTALL)/include/linux/limits.h: $(build)/$(linux_base_dir)/.canary
-	$(MAKE) \
-		-C "$(build)/$(linux_base_dir)" \
-		INSTALL_HDR_PATH="$(INSTALL)" \
-		O="$(linux_dir)" \
-		KCONFIG_CONFIG="$(pwd)/$(linux_kconfig)" \
-		headers_install
-

patches/make-4.2.1.patch

@@ -0,0 +1,11 @@
+--- clean/make-4.2/glob/glob.c	2013-10-20 17:14:38.000000000 +0000
++++ make-4.2/glob/glob.c	2018-09-18 10:16:03.860886356 +0000
+@@ -208,7 +208,7 @@
+ #endif /* __GNU_LIBRARY__ || __DJGPP__ */
+
+
+-#if !defined __alloca && !defined __GNU_LIBRARY__
++#if !defined __alloca && defined __GNU_LIBRARY__
+
+ # ifdef	__GNUC__
+ #  undef alloca