Merge pull request #6038 from ipfs/feat/gomod-yey

Introduce go modules [yey]

.circleci/config.yml

@@ -1,59 +1,62 @@
-version: 2
+version: 2.0
+
+aliases:
+  make_out_dirs: &make_out_dirs
+    run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS/{unit,sharness}
+  restore_gomod: &restore_gomod
+    restore_cache:
+      keys:
+        - v3-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-{{ .Environment.CIRCLE_JOB }}
+        - v3-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-
+        - v3-dep-{{ .Branch }}-
+        - v3-dep-master-
+  store_gomod: &store_gomod
+      save_cache:
+        key: v3-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-{{ .Environment.CIRCLE_JOB }}
+        paths:
+          - ~/go/pkg/mod
+          - ~/.cache/go-build/
+
 
 defaults: &defaults
   working_directory: ~/ipfs/go-ipfs
-  shell: /bin/bash --login
   environment:
+    GO111MODULE: "on"
+    TEST_NO_DOCKER: 1
     TEST_NO_FUSE: 1
-    GOPATH: $HOME/.go_workspace
+    GOPATH: /home/circleci/go
     CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
     CIRCLE: 1
     SERVICE: circle-ci
-    TEST_NO_DOCKER: 1
     CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
     TEST_VERBOSE: 1
     TRAVIS: 1
-    IMPORT_PATH: github.com/ipfs/go-ipfs
 
   docker:
-  - image: circleci/build-image:ubuntu-14.04-XXL-upstart-1189-5614f37
-    command: /sbin/init
+    - image: circleci/golang:1.11
 
 jobs:
   gotest:
     <<: *defaults
     steps:
     - checkout
-    - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
-    - run: 'sudo docker info >/dev/null 2>&1 || sudo service docker start; '
-    - run: sudo rm -rf /usr/local/go
-    - run: if [ ! -e go1.11.linux-amd64.tar.gz ]; then curl -o go1.11.linux-amd64.tar.gz https://storage.googleapis.com/golang/go1.11.linux-amd64.tar.gz; fi
-    - run: sudo tar -C /usr/local -xzf go1.11.linux-amd64.tar.gz
+    - *make_out_dirs
+    - *restore_gomod
 
-    - restore_cache:
-        keys:
-        - v1-dep-{{ .Branch }}-
-        - v1-dep-master-
+    - run: make cmd/ipfs-try-build
+    - run: |
+        make -j 1 test/unit/gotest.junit.xml \
+        && [[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]]
+    - run: 
+        when: always
+        command: bash <(curl -s https://codecov.io/bash) -cF unittests -X search -f coverage/unit_tests.coverprofile
 
-    - run: sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=0 net.ipv6.conf.default.disable_ipv6=0 net.ipv6.conf.all.disable_ipv6=0
-    - run: rm -rf "$HOME/.go_workspace/src/$IMPORT_PATH"
-    - run: mkdir -p "$HOME/.go_workspace/src/$IMPORT_PATH"
-    - run: cp -aT . "$HOME/.go_workspace/src/$IMPORT_PATH"
-    - run: cd "$HOME/.go_workspace/src/$IMPORT_PATH" && make deps
-
-    - save_cache:
-        key: v1-dep-{{ .Branch }}-{{ epoch }}
-        paths:
-        - ~/.go_workspace
-        - ~/go1.11.linux-amd64.tar.gz
-        - ~/.go_workspace/src/gx/ipfs
-        - ./node_modules
+    - run:
+        when: always
+        command: mv "test/unit/gotest.junit.xml" /tmp/circleci-test-results/unit
 
+    - *store_gomod
 
-    - run: cd "$HOME/.go_workspace/src/$IMPORT_PATH" && curl -s https://codecov.io/bash > codecov
-    - run: cd "$HOME/.go_workspace/src/$IMPORT_PATH" && make -j 1 test/unit/gotest.junit.xml
-    - run: cd "$HOME/.go_workspace/src/$IMPORT_PATH" && bash codecov -cF unittests -X search -f coverage/unit_tests.coverprofile
-    - run: mv "$HOME/.go_workspace/src/${IMPORT_PATH}/test/unit/gotest.junit.xml" /tmp/circleci-test-results
     - store_test_results:
         path: /tmp/circleci-test-results
     # Save artifacts
@@ -64,39 +67,23 @@ jobs:
   sharness:
     <<: *defaults
     steps:
+    - run: sudo apt install socat
     - checkout
-    - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
-    - run: 'sudo docker info >/dev/null 2>&1 || sudo service docker start; '
-    - run: sudo rm -rf /usr/local/go
-    - run: if [ ! -e go1.11.linux-amd64.tar.gz ]; then curl -o go1.11.linux-amd64.tar.gz https://storage.googleapis.com/golang/go1.11.linux-amd64.tar.gz; fi
-    - run: sudo tar -C /usr/local -xzf go1.11.linux-amd64.tar.gz
+    - *make_out_dirs
+    - *restore_gomod
 
-    - restore_cache:
-        keys:
-        - v1-dep-{{ .Branch }}-
-        - v1-dep-master-
+    - run: make -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1
 
-    - run: sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=0 net.ipv6.conf.default.disable_ipv6=0 net.ipv6.conf.all.disable_ipv6=0
-    - run: rm -rf "$HOME/.go_workspace/src/$IMPORT_PATH"
-    - run: mkdir -p "$HOME/.go_workspace/src/$IMPORT_PATH"
-    - run: cp -aT . "$HOME/.go_workspace/src/$IMPORT_PATH"
-    - run: cd "$HOME/.go_workspace/src/$IMPORT_PATH" && make deps
+    - run:
+        when: always
+        command: bash <(curl -s https://codecov.io/bash) -cF sharness -X search -f coverage/sharness_tests.coverprofile
 
-    - save_cache:
-        key: v1-dep-{{ .Branch }}-{{ epoch }}
-        paths:
-        - ~/.go_workspace
-        - ~/go1.11.linux-amd64.tar.gz
-        - ~/.go_workspace/src/gx/ipfs
-        - ./node_modules
+    - run: mv "test/sharness/test-results/sharness.xml" /tmp/circleci-test-results/sharness
+    # make sure we fail if there are test failures
+    - run: find test/sharness/test-results -name 't*-*.sh.*.counts' | test/sharness/lib/sharness/aggregate-results.sh | grep 'failed\s*0'
 
-    - run: cd "$HOME/.go_workspace/src/$IMPORT_PATH" && curl -s https://codecov.io/bash > codecov
-    - run: cd "$HOME/.go_workspace/src/$IMPORT_PATH" && make -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1
-    - run: cd "$HOME/.go_workspace/src/$IMPORT_PATH" && bash codecov -cF sharness -X search -f coverage/sharness_tests.coverprofile
-    - run: mv "$HOME/.go_workspace/src/${IMPORT_PATH}/test/sharness/test-results/sharness.xml" /tmp/circleci-test-results
+    - *store_gomod
 
-    # make sure we fail if there are test failures
-    - run: cd "$HOME/.go_workspace/src/$IMPORT_PATH" && ls test/sharness/test-results/t*-*.sh.*.counts | test/sharness/lib/sharness/aggregate-results.sh | grep 'failed\s*0'
     - store_test_results:
         path: /tmp/circleci-test-results
     # Save artifacts

.gitignore

@@ -17,6 +17,8 @@ gx-workspace-update.json
 bin/gx
 bin/gx*
 bin/tmp
+bin/gocovmerge
+bin/cover
 
 
 vendor

Rules.mk

@@ -12,7 +12,6 @@ include mk/git.mk # has to be before tarball.mk
 include mk/tarball.mk
 include mk/util.mk
 include mk/golang.mk
-include mk/gx.mk
 
 # -------------------- #
 #   extra properties   #
@@ -86,7 +85,7 @@ test: $(TEST)
 test_short: $(TEST_SHORT)
 .PHONY: test_short
 
-deps: gx-deps
+deps:
 .PHONY: deps
 
 nofuse: GOTAGS += nofuse

assets/assets.go

@@ -11,13 +11,13 @@ import (
 	"github.com/ipfs/go-ipfs/core"
 	"github.com/ipfs/go-ipfs/core/coreapi"
 
-	files "gx/ipfs/QmQmhotPUzVrMEWNK3x1R5jQ5ZHWyL7tVUrmRPjrBrvyCb/go-ipfs-files"
-	cid "gx/ipfs/QmTbxNB1NwDesLmKTscr4udL2tVP7MaxvXnD1D9yX7g3PN/go-cid"
-	iface "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core"
-	options "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core/options"
+	cid "github.com/ipfs/go-cid"
+	files "github.com/ipfs/go-ipfs-files"
+	iface "github.com/ipfs/interface-go-ipfs-core"
+	options "github.com/ipfs/interface-go-ipfs-core/options"
 
 	// this import keeps gx from thinking the dep isn't used
-	_ "gx/ipfs/QmT1jwrqzSMjSjLG5oBd9w4P9vXPKQksWuf5ghsE3Q88ZV/dir-index-html"
+	_ "github.com/ipfs/dir-index-html"
 )
 
 // initDocPaths lists the paths for the docs we want to seed during --init

bin/Rules.mk

@@ -2,9 +2,6 @@ include mk/header.mk
 
 dist_root_$(d)="/ipfs/QmPrXH9jRVwvd7r5MC5e6nV4uauQGzLk1i2647Ye9Vbbwe"
 
-$(d)/gx: $(d)/gx-v0.14.0
-$(d)/gx-go: $(d)/gx-go-v1.9.0
-
 TGTS_$(d) := $(d)/gx $(d)/gx-go
 DISTCLEAN += $(wildcard $(d)/gx-v*) $(wildcard $(d)/gx-go-v*) $(d)/tmp
 
@@ -18,13 +15,5 @@ else
 	ln -s $(notdir $^) $@
 endif
 
-bin/gx-v%:
-	@echo "installing gx $(@:bin/gx-%=%)"
-	bin/dist_get $(dist_root_bin) gx $@ $(@:bin/gx-%=%)
-
-bin/gx-go-v%:
-	@echo "installing gx-go $(@:bin/gx-go-%=%)"
-	@bin/dist_get $(dist_root_bin) gx-go $@ $(@:bin/gx-go-%=%)
-
 CLEAN += $(TGTS_$(d))
 include mk/footer.mk

bin/collect-profiles.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -x
+set -euo pipefail
+IFS=$'\n\t'
+
+HTTP_API="${1:-localhost:5001}"
+tmpdir=$(mktemp -d)
+export PPROF_TMPDIR="$tmpdir"
+pushd "$tmpdir"
+
+echo Collecting goroutine stacks
+curl -o goroutines.stacks "http://$HTTP_API"'/debug/pprof/goroutine?debug=2'
+
+echo Collecting goroutine profile
+go tool pprof -symbolize=remote -svg -output goroutine.svg "http://$HTTP_API/debug/pprof/goroutine"
+
+echo Collecting heap profile
+go tool pprof -symbolize=remote -svg -output heap.svg "http://$HTTP_API/debug/pprof/heap"
+
+echo "Collecting cpu profile (~30s)"
+go tool pprof -symbolize=remote -svg -output cpu.svg "http://$HTTP_API/debug/pprof/profile"
+
+echo "Enabling mutex profiling"
+curl -X POST -v "http://$HTTP_API"'/debug/pprof-mutex/?fraction=4'
+
+echo "Waiting for mutex data to be updated (30s)"
+sleep 30
+curl -o mutex.txt "http://$HTTP_API"'/debug/pprof/mutex?debug=2'
+go tool pprof -symbolize=remote -svg -output mutex.svg "http://$HTTP_API/debug/pprof/mutex"
+echo "Disabling mutex profiling"
+curl -X POST -v "http://$HTTP_API"'/debug/pprof-mutex/?fraction=0'
+
+popd
+tar cvzf "./ipfs-profile-$(uname -n)-$(date -Iseconds).tar.gz" -C "$tmpdir" .
+rm -rf "$tmpdir"
+
+
+
+

blocks/blockstoreutil/remove.go

@@ -7,9 +7,9 @@ import (
 
 	"github.com/ipfs/go-ipfs/pin"
 
-	cid "gx/ipfs/QmTbxNB1NwDesLmKTscr4udL2tVP7MaxvXnD1D9yX7g3PN/go-cid"
-	ds "gx/ipfs/QmUadX5EcvrBmxAV9sE7wUWtWSqxns5K84qKJBixmcT1w9/go-datastore"
-	bs "gx/ipfs/QmXjKkjMDTtXAiLBwstVexofB8LeruZmE2eBd85GwGFFLA/go-ipfs-blockstore"
+	cid "github.com/ipfs/go-cid"
+	ds "github.com/ipfs/go-datastore"
+	bs "github.com/ipfs/go-ipfs-blockstore"
 )
 
 // RemovedBlock is used to respresent the result of removing a block.

cmd/ipfs/Rules.mk

@@ -19,7 +19,7 @@ $(d)-try-build $(IPFS_BIN_$(d)): GOFLAGS += $(cmd/ipfs_flags)
 
 # uses second expansion to collect all $(DEPS_GO)
 $(IPFS_BIN_$(d)): $(d) $$(DEPS_GO) ALWAYS #| $(DEPS_OO_$(d))
-	$(go-build)
+	$(go-build-relative)
 
 TRY_BUILD_$(d)=$(addprefix $(d)-try-build-,$(SUPPORTED_PLATFORMS))
 $(d)-try-build: $(TRY_BUILD_$(d))