Fix linting/workflow errors in Github Actions

.github/workflows/compliance-test.yaml

@@ -0,0 +1,130 @@
+# Workflow for running the TES compliance suite against Funnel
+
+# This includes the following steps:
+#  1. Build Funnel and store the resulting binary artifact
+#  2. Install tes-compliance-suite and run against every version of TES simultaneously
+#  3. start-report-deployment: Send a dispatch to the funnel-compliance repository to generate and publish
+#     the tes-compliance-suite report to https://ohsu-comp-bio.github.io/funnel-compliance/
+
+# Optionally debug via SSH
+# Ref: https://fleetdm.com/engineering/tips-for-github-actions-usability
+#
+# To use this step uncomment and place anywhere in the build steps. The build will pause on this step and
+# output a ssh address associated with the Github action worker. Helpful for debugging build steps and
+# and intermediary files/artifacts.
+#
+# Example:
+#   Web shell: https://tmate.io/t/q8FU3U9SvmMVxAhMHRyExNhr8
+#   SSH: ssh [email protected]
+#
+# - name: "Debug: Package dependancies for tmate (CentOS)"
+#   run: |
+#     yum install -y xz
+#     ln -s /bin/true /bin/apt-get
+#
+# - name: Setup tmate session
+#   uses: mxschmitt/action-tmate@v3
+
+name: Compliance Test
+
+on:
+  push:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    container: quay.io/ohsu-comp-bio/slurm
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: ^1.18
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Build
+      run: make build
+
+    - name: Store funnel
+      uses: actions/upload-artifact@v2
+      with:
+        name: funnelBin
+        path: funnel
+
+  compliance:
+    strategy:
+      fail-fast: false
+      matrix:
+        version: [1.0.0, 1.1.0]
+        db: ["boltdb", "mongodb"]
+        compute: ["local", "slurm"]
+    needs: build
+    runs-on: ubuntu-latest
+    container:
+      image: quay.io/ohsu-comp-bio/slurm
+      options: --hostname slurmctl --cap-add sys_admin
+    steps:
+      # Required to access the 'tests/mongo.config.yml' file
+      # Perhaps uploading it as an artifact would be more efficient?
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: funnelBin
+
+      - name: Start Funnel server
+        run: |
+          touch config.yml
+          if [ ${{ matrix.db }} = "mongodb" ]; then
+            make start-mongodb
+            cat `pwd`/tests/mongo.config.yml >> config.yml
+            # Required for Funnel to connect MongoDB
+            echo "172.17.0.1 localhost" >> /etc/hosts
+          elif [ ${{ matrix.compute }} = "slurm" ]; then
+            cat `pwd`/tests/slurm.config.yml >> config.yml
+            cp config.yml /opt/funnel_config.yml
+            # Start Slurm
+            /usr/local/bin/docker-entrypoint.sh
+          fi
+          chmod +x funnel
+          FLAGS="--config `pwd`/config.yml"
+          ./funnel server run $FLAGS &> funnel.logs &
+
+      - uses: actions/checkout@v3
+        with:
+          repository: lbeckman314/tes-compliance-suite
+          ref: feature/tesv1.1
+          path: tes-compliance-suite
+
+      - name: Install TES compliance suite
+        run: |
+          cd tes-compliance-suite
+          /root/.pyenv/shims/python3 -m venv venv
+          source venv/bin/activate
+          pip install -r requirements.txt
+          python setup.py install
+          mkdir reports
+
+      - name: Test compliance
+        run: |
+          cd tes-compliance-suite
+          source venv/bin/activate
+          tes-compliance-suite report --version "${{ matrix.version }}" --server "http://localhost:8000/"
+
+  start-report-deployment:
+    needs: compliance
+    runs-on: ubuntu-latest
+    steps:
+      # https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#create-a-repository-dispatch-event
+      - name: Start report generation
+        uses: passeidireto/trigger-external-workflow-action@main
+        env:
+          PAYLOAD_AUTHOR: "Funnel"
+          PAYLOAD_REVISION: "3"
+        with:
+          repository: ohsu-comp-bio/funnel-compliance
+          event: start-report
+          github_pat: ${{ secrets.ACTIONS_TOKEN }}
+

.github/workflows/tests.yaml

@@ -1,6 +1,8 @@
 name: Go
 
-on: [ pull_request ]
+on:
+  - pull_request
+  - push
 
 jobs:
 
@@ -15,8 +17,8 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.52.2
-          args: --timeout 3m --verbose
+          version: latest
+          args: --timeout 3m --verbose -D unused -D errcheck -D staticcheck -D govet -D gosimple -D ineffassign
 
   build:
     runs-on: ubuntu-latest
@@ -30,7 +32,7 @@ jobs:
       uses: actions/checkout@v2
 
     - name: Build
-      run: go build -v ./
+      run: make build
 
     - name: Store funnel
       uses: actions/upload-artifact@v2
@@ -47,8 +49,9 @@ jobs:
         go-version: ^1.18
     - name: Check out code
       uses: actions/checkout@v2
+
     - name: Unit Tests
-      run: make test
+      run: make test-verbose
 
   mongoTest:
     runs-on: ubuntu-latest
@@ -61,6 +64,7 @@ jobs:
       uses: actions/download-artifact@v2
       with:
         name: funnelBin
+
     - name: MongoTest
       run: |
         chmod +x funnel
@@ -83,3 +87,39 @@ jobs:
       run: |
         chmod +x funnel
         make test-badger
+
+  slurmTest:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Download funnel bin
+      uses: actions/download-artifact@v2
+      with:
+        name: funnelBin
+
+    - name: Slurm Test
+      run: |
+        chmod +x funnel
+        make test-slurm
+
+  s3Test:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Download funnel bin
+      uses: actions/download-artifact@v2
+      with:
+        name: funnelBin
+
+    - name: S3 Test
+      run: |
+        chmod +x funnel
+        make start-generic-s3
+        sleep 10
+        make test-generic-s3

.gitignore

@@ -21,3 +21,5 @@ docker/funnel
 deployments/kubernetes/**/funnel
 deployments/kubernetes/**/*.json
 funnel
+dist/
+.vscode

.goreleaser.yml → .goreleaser.yaml

@@ -5,6 +5,7 @@ builds:
       - linux
     goarch:
       - amd64
+      - arm64
     env:
       - CGO_ENABLED=0
     ldflags: >
@@ -27,7 +28,7 @@ archives:
     name_template: "{{.ProjectName}}-{{.Os}}-{{.Arch}}-{{.Version}}"
 
 brews:
-  - github:
+  - tap:
       owner: ohsu-comp-bio
       name: homebrew-formula
     folder: Formula

.vscode/launch.json

@@ -0,0 +1,20 @@
+{
+    "configurations": [
+        {
+            "name": "CLI",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "main.go",
+            "args": ["server", "run"],
+        },
+        {
+            "name": "Attach to Process",
+            "type": "go",
+            "request": "attach",
+            "mode": "local",
+            "justMyCode": true,
+            "processId": "${command:pickProcess}",
+        },
+    ]
+}

Makefile

@@ -11,11 +11,11 @@ git_upstream := $(shell git remote get-url $(shell git config branch.$(shell git
 export GIT_BRANCH = $(git_branch)
 export GIT_UPSTREAM = $(git_upstream)
 
-export FUNNEL_VERSION=0.10.1
+export FUNNEL_VERSION=0.11.0-rc.1
 
 # LAST_PR_NUMBER is used by the release notes builder to generate notes
 # based on pull requests (PR) up until the last release.
-export LAST_PR_NUMBER = 605
+export LAST_PR_NUMBER = 716
 
 VERSION_LDFLAGS=\
  -X "github.com/ohsu-comp-bio/funnel/version.BuildDate=$(shell date)" \
@@ -33,7 +33,13 @@ install:
 # Build the code
 build:
 	@touch version/version.go
-	@go build -ldflags '$(VERSION_LDFLAGS)' .
+	@go build -ldflags '$(VERSION_LDFLAGS)' -buildvcs=false .
+
+# Build an unoptimized version of the code for use during debugging 
+# https://go.dev/doc/gdb
+debug:
+	@go install -gcflags=all="-N -l"
+	@funnel server run
 
 # Generate the protobuf/gRPC code
 proto:
@@ -70,6 +76,15 @@ proto:
 		--grpc-gateway_opt paths=source_relative \
 		events.proto
 
+
+proto-depends:
+	@git submodule update --init --recursive
+	@go install github.com/grpc-ecosystem/grpc-gateway/v2/[email protected]
+	@go install github.com/grpc-ecosystem/grpc-gateway/v2/[email protected]
+	@go install google.golang.org/protobuf/cmd/[email protected]
+	@go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
+	@go install github.com/ckaznocha/[email protected]
+
 # Start API reference doc server
 serve-doc:
 	@go get golang.org/x/tools/cmd/godoc
@@ -87,7 +102,7 @@ lint-depends:
 
 # Run code style and other checks
 lint:
-	@golangci-lint run --timeout 3m --disable-all --enable=golint --enable=gofmt --enable=goimports --enable=misspell \
+	@golangci-lint run --timeout 3m --disable-all --enable=vet --enable=golint --enable=gofmt --enable=goimports --enable=misspell \
 		--skip-dirs "vendor" \
 		--skip-dirs "webdash" \
 		--skip-dirs "cmd/webdash" \
@@ -113,7 +128,7 @@ test-elasticsearch:
 
 start-mongodb:
 	@docker rm -f funnel-mongodb-test > /dev/null 2>&1 || echo
-	@docker run -d --name funnel-mongodb-test -p 27000:27017 docker.io/mongo:3.5.13 > /dev/null
+	@docker run -d --name funnel-mongodb-test -p 27000:27017 docker.io/mongo > /dev/null
 
 test-mongodb:
 	@go test ./tests/core/ --funnel-config `pwd`/tests/mongo.config.yml
@@ -149,7 +164,7 @@ test-htcondor:
 	@go test -timeout 120s ./tests/htcondor -funnel-config `pwd`/tests/htcondor.config.yml
 
 test-slurm:
-	@docker pull ohsucompbio/slurm
+	@docker pull quay.io/ohsu-comp-bio/slurm
 	@go test -timeout 120s ./tests/slurm -funnel-config `pwd`/tests/slurm.config.yml
 
 test-gridengine:
@@ -165,9 +180,9 @@ test-amazon-s3:
 
 start-generic-s3:
 	@docker rm -f funnel-s3server > /dev/null 2>&1 || echo
-	@docker run -d --name funnel-s3server -p 18888:8000 scality/s3server:mem-6018536a
+	@docker run -d --name funnel-s3server -p 18888:8000 zenko/cloudserver
 	@docker rm -f funnel-minio > /dev/null 2>&1 || echo
-	@docker run -d --name funnel-minio -p 9000:9000 -e "MINIO_ACCESS_KEY=fakekey" -e "MINIO_SECRET_KEY=fakesecret" -e "MINIO_REGION=us-east-1" minio/minio:RELEASE.2017-10-27T18-59-02Z server /data
+	@docker run -d --name funnel-minio -p 9000:9000 -e "MINIO_ACCESS_KEY=fakekey" -e "MINIO_SECRET_KEY=fakesecret" -e "MINIO_REGION=us-east-1" minio/minio server /data
 
 test-generic-s3:
 	@go test ./tests/storage -funnel-config `pwd`/tests/amazoncli-minio-s3.config.yml -run TestAmazonS3Storage
@@ -202,9 +217,9 @@ webdash:
 	@go-bindata -pkg webdash -prefix "webdash/build" -o webdash/web.go webdash/build/...
 
 # Build binaries for all OS/Architectures
-snapshot: depends
+snapshot: release-dep
 	@goreleaser \
-		--rm-dist \
+		--clean \
 		--snapshot
 
 # build a docker container locally
@@ -219,12 +234,14 @@ docker-dind:
 docker-dind-rootless:
 	docker build -t ohsucompbio/funnel-dind-rootless:latest -f Dockerfile.dind-rootless .
 
+# Create a release on Github using GoReleaser 
 release:
 	@go get github.com/buchanae/github-release-notes
 	@goreleaser \
-		--rm-dist \
+		--clean \
 		--release-notes <(github-release-notes -org ohsu-comp-bio -repo funnel -stop-at ${LAST_PR_NUMBER})
 
+# Install dependencies for release
 release-dep:
 	@go get github.com/goreleaser/goreleaser
 	@go get github.com/buchanae/github-release-notes
@@ -260,4 +277,4 @@ website-dev:
 clean:
 	@rm -rf ./bin ./pkg ./test_tmp ./build ./buildtools
 
-.PHONY: proto website docker webdash build
+.PHONY: proto website docker webdash build debug

cmd/run/cmd.go

@@ -22,7 +22,8 @@ var Cmd = &cobra.Command{
 	RunE: func(cmd *cobra.Command, args []string) error {
 		err := Run(args)
 		if err != nil {
-			//cmd.Usage()
+			cmd.Usage()
+			return err
 		}
 		return err
 	},