Static build and cross-compilation with Docker root-gg#331

.travis.yml

@@ -7,13 +7,8 @@ go:
 
 go_import_path: github.com/root-gg/plik
 
-before_install:
-  # Update node_js version : https://github.com/travis-ci/travis-ci/issues/7108
-  - rm -rf ~/.nvm && git clone https://github.com/creationix/nvm.git ~/.nvm && (cd ~/.nvm && git checkout `git describe --abbrev=0 --tags`) && source ~/.nvm/nvm.sh && nvm install stable
-  - npm install -g bower
-
 before_script:
-  - go get -u golang.org/x/lint/golint
+  - GO111MODULE=off go get -u golang.org/x/lint/golint
 
 script:
-  - make && make lint && make test && make test-backends && make docker
+  - make lint && make test && make test-backends && make release

Dockerfile

@@ -1,6 +1,5 @@
 ##################################################################################
-# Builder 1: make frontend
-FROM node:12.15-alpine AS builder-frontend
+FROM node:12.15-alpine AS plik-frontend-builder
 
 # Install needed binaries
 RUN apk add --no-cache git make bash
@@ -9,14 +8,16 @@ RUN apk add --no-cache git make bash
 ADD Makefile .
 ADD webapp /webapp
 
-RUN make frontend
+##################################################################################
+FROM plik-frontend-builder AS plik-frontend
+
+RUN make clean-frontend frontend
 
 ##################################################################################
-# Builder 2: make server and clients
-FROM golang:1.15.2-alpine AS builder-go
+FROM golang:1.15.2-buster AS plik-builder
 
 # Install needed binaries
-RUN apk add --no-cache git make bash gcc g++
+RUN apt-get update && apt-get install -y build-essential crossbuild-essential-armhf crossbuild-essential-armel crossbuild-essential-arm64 crossbuild-essential-i386
 
 # Prepare the source location
 RUN mkdir -p /go/src/github.com/root-gg/plik
@@ -25,23 +26,23 @@ WORKDIR /go/src/github.com/root-gg/plik
 # Add the source code ( see .dockerignore )
 ADD . .
 
-# Build clients ( all arch )
-RUN make clients
+# Copy webapp build from previous stage
+COPY --from=plik-frontend /webapp/dist webapp/dist
+
+##################################################################################
+FROM plik-builder AS plik-releases
+
+ARG CLIENT_TARGETS=""
+ENV CLIENT_TARGETS=$CLIENT_TARGETS
 
-# Set server target arch
-ARG GOOS=""
-ARG GOARCH=""
-ENV GOOS=${GOOS}
-ENV GOARCH=${GOARCH}
+ARG TARGETS=""
+ENV TARGETS=$TARGETS
 
-# Build server
-RUN make server
+RUN releaser/releaser.sh
 
 ##################################################################################
-# Builder 3: we need ca-certificates in the final container
-FROM alpine:3.11 AS builder-env
+FROM alpine:3.12 AS plik-base
 
-# Install needed binaries
 RUN apk add --no-cache ca-certificates
 
 # Create plik user
@@ -57,14 +58,27 @@ RUN adduser \
     --uid "${UID}" \
     "${USER}"
 
-COPY --from=builder-frontend --chown=1000:1000 /webapp/dist /home/plik/webapp/dist
-
-COPY --from=builder-go --chown=1000:1000 /go/src/github.com/root-gg/plik/server/plikd /home/plik/server/plikd
-COPY --from=builder-go --chown=1000:1000 /go/src/github.com/root-gg/plik/server/plikd.cfg /home/plik/server/plikd.cfg
-COPY --from=builder-go --chown=1000:1000 /go/src/github.com/root-gg/plik/clients /home/plik/clients
-COPY --from=builder-go --chown=1000:1000 /go/src/github.com/root-gg/plik/changelog /home/plik/changelog
-
 EXPOSE 8080
 USER plik
 WORKDIR /home/plik/server
-CMD ./plikd
+CMD ./plikd
+
+##################################################################################
+FROM plik-base AS plik-amd64
+
+COPY --from=plik-releases --chown=1000:1000 /go/src/github.com/root-gg/plik/releases/plik-*-linux-amd64 /home/plik/
+
+##################################################################################
+FROM plik-base AS plik-386
+
+COPY --from=plik-releases --chown=1000:1000 /go/src/github.com/root-gg/plik/releases/plik-*-linux-386 /home/plik/
+
+##################################################################################
+FROM plik-base AS plik-arm
+
+COPY --from=plik-releases --chown=1000:1000 /go/src/github.com/root-gg/plik/releases/plik-*-linux-arm /home/plik/
+
+##################################################################################
+FROM plik-base AS plik-arm64
+
+COPY --from=plik-releases --chown=1000:1000 /go/src/github.com/root-gg/plik/releases/plik-*-linux-arm64 /home/plik/

Makefile

@@ -1,22 +1,11 @@
 SHELL = /bin/bash
 
-RELEASE_VERSION = $(shell version/version.sh)
-RELEASE_DIR = "release/plik-$(RELEASE_VERSION)"
-RELEASE_TARGETS = darwin-amd64 freebsd-386 \
-freebsd-amd64 linux-386 linux-amd64 linux-arm openbsd-386 \
-openbsd-amd64 windows-amd64 windows-386
+BUILD_INFO = $(shell server/gen_build_info.sh base64)
+BUILD_FLAG = -ldflags="-X github.com/root-gg/plik/server/common.buildInfoString=$(BUILD_INFO) -w -s -extldflags=-static"
+BUILD_TAGS = -tags osusergo,netgo,sqlite_omit_load_extension
 
-GOHOSTOS = $(if $(GOOS),$(GOOS),$(shell go env GOHOSTOS))
-GOHOSTARCH = $(if $(GOARCH),$(GOARCH),$(shell go env GOHOSTARCH))
-
-DEBROOT_SERVER = debs/server
-DEBROOT_CLIENT = debs/client
-
-BUILD_INFO = $(shell server/gen_build_info.sh $(RELEASE_VERSION) base64)
-BUILD_FLAG = -ldflags="-X github.com/root-gg/plik/server/common.buildInfoString=$(BUILD_INFO)"
-
-GO_BUILD = go build $(BUILD_FLAG)
-GO_TEST = GORACE="halt_on_error=1" go test $(BUILD_FLAG) -race -cover -p 1
+GO_BUILD = go build $(BUILD_FLAG) $(BUILD_TAGS)
+GO_TEST = GORACE="halt_on_error=1" go test $(BUILD_FLAG) $(BUILD_TAGS) -race -cover -p 1
 
 ifdef ENABLE_RACE_DETECTOR
 	GO_BUILD := GORACE="halt_on_error=1" $(GO_BUILD) -race
@@ -36,158 +25,36 @@ frontend:
 # Build plik server for the current architecture
 ###
 server:
-	@server/gen_build_info.sh $(RELEASE_VERSION) info
+	@server/gen_build_info.sh info
 	@echo "Building Plik server"
 	@cd server && $(GO_BUILD) -o plikd
 
-###
-# Build plik server for all architectures
-###
-servers: frontend
-	@server/gen_build_info.sh $(RELEASE_VERSION) info
-	@cd server && for target in $(RELEASE_TARGETS) ; do \
-		SERVER_DIR=../servers/$$target; \
-		SERVER_PATH=$$SERVER_DIR/plikd;  \
-		export GOOS=`echo $$target | cut -d "-" -f 1`; 	\
-		export GOARCH=`echo $$target | cut -d "-" -f 2`; \
-		mkdir -p ../servers/$$target; \
-		if [ $$GOOS = "windows" ] ; then SERVER_PATH=$$SERVER_DIR/plikd.exe ; fi ; \
-		if [ -e $$SERVER_PATH ] ; then continue ; fi ; \
-		echo "Building Plik server for $$target to $$SERVER_PATH"; \
-		$(GO_BUILD) -o $$SERVER_PATH ;	\
-	done
-
 ###
 # Build plik client for the current architecture
 ###
 client:
-	@server/gen_build_info.sh $(RELEASE_VERSION) info
+	@server/gen_build_info.sh info
 	@echo "Building Plik client"
 	@cd client && $(GO_BUILD) -o plik ./
 
-
 ###
-# Build plik client for all architectures
+# Build clients for all architectures
 ###
 clients:
-	@server/gen_build_info.sh $(RELEASE_VERSION) info
-	@cd client && for target in $(RELEASE_TARGETS) ; do	\
-		CLIENT_DIR=../clients/$$target;	\
-		CLIENT_PATH=$$CLIENT_DIR/plik;	\
-		CLIENT_MD5=$$CLIENT_DIR/MD5SUM;	\
-		export GOOS=`echo $$target | cut -d "-" -f 1`; \
-		export GOARCH=`echo $$target | cut -d "-" -f 2`; \
-		mkdir -p $$CLIENT_DIR; \
-		if [ $$GOOS = "windows" ] ; then CLIENT_PATH=$$CLIENT_DIR/plik.exe ; fi ; \
-		if [ -e $$CLIENT_PATH ] ; then continue ; fi ; \
-		echo "Building Plik client for $$target to $$CLIENT_PATH"; \
-		$(GO_BUILD) -o $$CLIENT_PATH ; \
-		md5sum $$CLIENT_PATH | awk '{print $$1}' > $$CLIENT_MD5; \
-	done
-	@mkdir -p clients/bash && cp client/plik.sh clients/bash
-
-###
-# Make server and clients Debian packages
-###
-debs: debs-client debs-server
+	# Only build clients
+	@TARGETS="skip" releaser/releaser.sh
 
 ###
-# Make server Debian packages
-###
-debs-server: servers clients
-	@mkdir -p $(DEBROOT_SERVER)/usr/local/plikd/server
-	@mkdir -p $(DEBROOT_SERVER)/etc/init.d
-	@cp -R server/build/deb/DEBIAN $(DEBROOT_SERVER)
-	@cp -R clients/ $(DEBROOT_SERVER)/usr/local/plikd/clients
-	@cp -R server/public/ $(DEBROOT_SERVER)/usr/local/plikd/server/public
-	@cp -R server/plikd.cfg $(DEBROOT_SERVER)/etc/plikd.cfg
-	@cp -R server/plikd.init $(DEBROOT_SERVER)/etc/init.d/plikd && chmod +x $(DEBROOT_SERVER)/etc/init.d/plikd
-	@for arch in amd64 i386 armhf ; do \
-		cp -R server/build/deb/DEBIAN/control $(DEBROOT_SERVER)/DEBIAN/control ; \
-		sed -i -e "s/##ARCH##/$$arch/g" $(DEBROOT_SERVER)/DEBIAN/control ; \
-		sed -i -e "s/##VERSION##/$(RELEASE_VERSION)/g" $(DEBROOT_SERVER)/DEBIAN/control ; \
-		if [ $$arch = 'i386' ]; then \
-			cp servers/linux-386/plikd $(DEBROOT_SERVER)/usr/local/plikd/server/ ; \
-		elif [ $$arch = 'armhf' ]; then  \
-			cp servers/linux-arm/plikd $(DEBROOT_SERVER)/usr/local/plikd/server/ ; \
-		else \
-			cp servers/linux-$$arch/plikd $(DEBROOT_SERVER)/usr/local/plikd/server/ ; \
-		fi ; \
-		dpkg-deb --build $(DEBROOT_SERVER) debs/plikd-$(RELEASE_VERSION)-$$arch.deb ; \
-	done
-
+# Display build info
 ###
-# Make client Debian packages
-###
-debs-client: clients
-	@mkdir -p $(DEBROOT_CLIENT)/usr/local/bin
-	@cp -R client/build/deb/DEBIAN $(DEBROOT_CLIENT)
-	@for arch in amd64 i386 armhf ; do \
-		cp -R client/build/deb/DEBIAN/control $(DEBROOT_CLIENT)/DEBIAN/control ; \
-		sed -i -e "s/##ARCH##/$$arch/g" $(DEBROOT_CLIENT)/DEBIAN/control ; \
-		sed -i -e "s/##VERSION##/$(RELEASE_VERSION)/g" $(DEBROOT_CLIENT)/DEBIAN/control ; \
-		if [ $$arch = 'i386' ]; then \
-			cp clients/linux-386/plik $(DEBROOT_CLIENT)/usr/local/bin ; \
-		elif [ $$arch = 'armhf' ]; then  \
-			cp clients/linux-arm/plik $(DEBROOT_CLIENT)/usr/local/bin ; \
-		else \
-			cp clients/linux-$$arch/plik $(DEBROOT_CLIENT)/usr/local/bin ; \
-		fi ; \
-		dpkg-deb --build $(DEBROOT_CLIENT) debs/plik-$(RELEASE_VERSION)-$$arch.deb ; \
-	done
-
-###
-# Prepare the release base (css, js, ...)
-###
-release-template: clean frontend clients
-	@mkdir -p $(RELEASE_DIR)/webapp
-	@mkdir -p $(RELEASE_DIR)/server
-	@cp -r clients $(RELEASE_DIR)
-	@cp -r changelog $(RELEASE_DIR)
-	@cp -r webapp/dist $(RELEASE_DIR)/webapp/dist
-	@cp -r server/plikd.cfg $(RELEASE_DIR)/server
-
-###
-# Build release archive
-###
-release: release-template server
-	@cp -R server/plikd $(RELEASE_DIR)/server/plikd
-	@cd release && tar czvf plik-$(RELEASE_VERSION)-$(GOHOSTOS)-$(GOHOSTARCH).tar.gz plik-$(RELEASE_VERSION)
-
-###
-# Build release archives for all architectures
-###
-releases: release-template servers
-
-	@mkdir -p releases
-
-	@cd release && for target in $(RELEASE_TARGETS) ; do \
-		SERVER_PATH=../servers/$$target/plikd;  \
-		OS=`echo $$target | cut -d "-" -f 1`; \
-		ARCH=`echo $$target | cut -d "-" -f 2`; \
-		if [ $$OS = "darwin" ] ; then OS="macos" ; fi ; \
-		if [ $$OS = "windows" ] ; then SERVER_PATH=../servers/$$target/plikd.exe ; fi ; \
-		if [ $$ARCH = "386" ] ; then ARCH="32bits" ; fi ; \
-		if [ $$ARCH = "amd64" ] ; then ARCH="64bits" ; fi ; \
-		cp -R $$SERVER_PATH plik-$(RELEASE_VERSION)/server; \
-		if [ $$OS = "windows" ] ; then \
-			TARBALL_NAME=plik-$(RELEASE_VERSION)-$$OS-$$ARCH.zip; \
-			echo "Packaging plik release for $$target to $$TARBALL_NAME"; \
-			zip -r ../releases/$$TARBALL_NAME plik-$(RELEASE_VERSION); \
-		else \
-			TARBALL_NAME=plik-$(RELEASE_VERSION)-$$OS-$$ARCH.tar.gz; \
-			echo "Packaging plik release for $$target to $$TARBALL_NAME"; \
-			tar czvf ../releases/$$TARBALL_NAME plik-$(RELEASE_VERSION); \
-		fi \
-	done
-
-	@md5sum releases/* > releases/md5sum.txt
+build-info:
+	@server/gen_build_info.sh info
 
 ###
-# Generate build info
+# Display version
 ###
-build-info:
-	@server/gen_build_info.sh $(RELEASE_VERSION) info
+version:
+	@server/gen_build_info.sh version
 
 ###
 # Run linters
@@ -222,21 +89,16 @@ test-backends:
 	@testing/test_backends.sh
 
 ###
-# Build docker
+# release
 ###
-docker:
-	docker build --build-arg "GOOS=$$GOOS" --build-arg "GOARCH=$$GOARCH" -t rootgg/plik-$(GOHOSTOS)-$(GOHOSTARCH):$(RELEASE_VERSION) .
+release:
+	@releaser/release.sh
 
 ###
-# Build dockers
+# docker
 ###
-dockers:
-	@for target in $(RELEASE_TARGETS) ; do \
-		GOOS=`echo $$target | cut -d "-" -f 1`; \
-		GOARCH=`echo $$target | cut -d "-" -f 2`; \
-		docker build --build-arg "GOOS=$$GOOS" --build-arg "GOARCH=$$GOARCH" -t rootgg/plik-$$target:$(RELEASE_VERSION) . ; \
-	done
-
+docker:
+	@docker build -t rootgg/plik:dev --build-arg "TARGETS=amd64" --target="plik-amd64" .
 
 ###
 # Remove server build files
@@ -246,7 +108,6 @@ clean:
 	@rm -rf client/plik
 	@rm -rf clients
 	@rm -rf servers
-	@rm -rf debs
 	@rm -rf release
 	@rm -rf releases
 
@@ -268,4 +129,4 @@ clean-all: clean clean-frontend
 # by make, we must declare these targets as phony to avoid :
 # "make: `client' is up to date" cases at compile time
 ###
-.PHONY: client server
+.PHONY: client clients server release

README.md

@@ -354,14 +354,25 @@ Please be sure to also run/update the test suite :
 
 * Cross compilation
 
-To target a specific architecture :
+All binary are now statically linked  
+Clients can be safely cross-compiled for all os/architectures as they do not rely on GCO (sqlite)
 ```
-    GOOS=linux GOARCH=arm make server
-    GOOS=linux GOARCH=arm make client
-    GOOS=linux GOARCH=arm make docker
+    GOOS=windows GOARCH=amd64 make client
 ```
 
-The `make releases` target build a release package for each architecture specified in Makefile  
-The `make dockers` target build a docker image for each architecture specified in Makefile  
-The `make clients` target build the plik clients for each architecture specified in Makefile  
-The `make servers` target build the plik servers for each architecture specified in Makefile  
+Servers rely on CGO/sqlite so we cross-compile it for Linux only using Docker.  
+The `make release` target will build a release package and Docker images for `amd64,i386,arm,arm64`
+
+If you want a more specific ARMv7 for hardware floating point build for example
+See : https://github.com/golang/go/wiki/GoArm
+```
+    make release
+    docker run -it rootgg/plik-builder:latest /bin/bash
+
+    GOOS=linux GOARCH=arm GOARM=7 CGO_ENABLED=1 CC=arm-linux-gnueabihf-gcc make server
+
+    file server/plikd
+    server/plikd: ELF 32-bit LSB executable, ARM, EABI5 version 1 (GNU/Linux), statically linked, for GNU/Linux 3.2.0, Go ...
+
+    Then either copy the binary from the docker or play with releaser/releaser.sh to generate a release archive
+```