Add initial Nextflow tests (nf-canary)

.github/workflows/build.yaml

@@ -0,0 +1,26 @@
+name: Build
+
+on:
+  - pull_request
+  - push
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: ^1.18
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Build
+      run: make build
+
+    - name: Store funnel
+      uses: actions/upload-artifact@v2
+      with:
+        name: funnelBin
+        path: funnel

.github/workflows/compliance-test.yaml

@@ -28,38 +28,19 @@
 name: Compliance Test
 
 on:
-  push:
+  workflow_run:
+    workflows: ["Build"]
+    types:
+      - completed
 
 jobs:
-  build:
-    runs-on: ubuntu-latest
-    container: quay.io/ohsu-comp-bio/slurm
-    steps:
-    - name: Set up Go 1.x
-      uses: actions/setup-go@v2
-      with:
-        go-version: ^1.18
-
-    - name: Check out code
-      uses: actions/checkout@v2
-
-    - name: Build
-      run: make build
-
-    - name: Store funnel
-      uses: actions/upload-artifact@v2
-      with:
-        name: funnelBin
-        path: funnel
-
   compliance:
     strategy:
       fail-fast: false
       matrix:
         version: [1.0.0, 1.1.0]
         db: ["boltdb", "mongodb"]
         compute: ["local", "slurm"]
-    needs: build
     runs-on: ubuntu-latest
     container:
       image: quay.io/ohsu-comp-bio/slurm

.github/workflows/tests.yaml

@@ -1,11 +1,12 @@
 name: Go
 
 on:
-  - pull_request
-  - push
+  workflow_run:
+    workflows: ["Build"]
+    types:
+      - completed
 
 jobs:
-
   lint:
     name: lint
     runs-on: ubuntu-latest
@@ -20,26 +21,6 @@ jobs:
           version: latest
           args: --timeout 3m --verbose -D unused -D errcheck -D staticcheck -D govet -D gosimple -D ineffassign
 
-  build:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Set up Go 1.x
-      uses: actions/setup-go@v2
-      with:
-        go-version: ^1.18
-
-    - name: Check out code
-      uses: actions/checkout@v2
-
-    - name: Build
-      run: make build
-
-    - name: Store funnel
-      uses: actions/upload-artifact@v2
-      with:
-        name: funnelBin
-        path: funnel
-
   unitTest:
     runs-on: ubuntu-latest
     steps:
@@ -55,7 +36,6 @@ jobs:
 
   mongoTest:
     runs-on: ubuntu-latest
-    needs: build
     steps:
     - name: Check out code
       uses: actions/checkout@v2
@@ -74,7 +54,6 @@ jobs:
 
   badgerTest:
     runs-on: ubuntu-latest
-    needs: build
     steps:
     - name: Check out code
       uses: actions/checkout@v2
@@ -90,7 +69,6 @@ jobs:
 
   slurmTest:
     runs-on: ubuntu-latest
-    needs: build
     steps:
     - name: Check out code
       uses: actions/checkout@v2
@@ -107,7 +85,6 @@ jobs:
 
   s3Test:
     runs-on: ubuntu-latest
-    needs: build
     steps:
     - name: Check out code
       uses: actions/checkout@v2
@@ -123,3 +100,18 @@ jobs:
         make start-generic-s3
         sleep 10
         make test-generic-s3
+
+  nextflow:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Download funnel
+      uses: actions/download-artifact@v2
+      with:
+        name: funnelBin
+        path: funnel
+
+    - name: Run Nextflow test (nf-canary)
+      run: funnel task create examples/nextflow.json

.github/workflows/workflows.yaml

@@ -0,0 +1,28 @@
+name: Workflow Tests
+
+# Only trigger, when the build workflow succeeded
+on:
+  workflow_run:
+    workflows: ["Build"]
+    types:
+      - completed
+
+jobs:
+  nextflow:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Download funnel
+      uses: actions/download-artifact@v2
+      with:
+        name: funnelBin
+        path: funnel
+
+    - name: Run Nextflow test (nf-canary)
+      run: |
+        chmod +x funnel
+        funnel server run --config funnel.config
+        funnel task create examples/nextflow.json
+    

examples/nextflow.json

@@ -0,0 +1,26 @@
+{
+  "name": "Nextflow Canary Tests",
+  "description": "Run the Nextflow Canary tests to verify Nextflow support",
+  "tags": {
+    "Project": "Nextflow"
+  },
+  "outputs": [
+    {
+      "url": "/tmp",
+      "path": "/.nextflow/",
+      "type": "DIRECTORY"
+    },
+    {
+      "url": "/tmp",
+      "path": "/root/",
+      "type": "DIRECTORY"
+    }
+  ],
+  "executors": [
+    {
+      "image": "quay.io/ohsu-comp-bio/nf-canary",
+      "command": ["sh", "-c", "cd /root; nextflow -Djdk.lang.Process.launchMechanism=vfork run /nf-canary/main.nf"]
+    }
+  ]
+}
+

funnel.config

@@ -0,0 +1,9 @@
+LocalStorage:
+  # Whitelist of local directory paths which Funnel is allowed to access.
+  AllowedDirs:
+    - ./
+    - /tmp
+
+Worker:
+  LeaveWorkDir: true
+

storage/local.go

@@ -181,6 +181,24 @@ func linkFile(ctx context.Context, source string, dest string) error {
 	if same {
 		return nil
 	}
+	if source != parent {
+		fileInfo, err := os.Stat(parent)
+		if err != nil{
+			return err
+		}
+		if fileInfo.IsDir() {
+			fmt.Printf("DEBUG: %v is a symlink to directory %v.\n", source, parent)
+			fmt.Printf("DEBUG dest: %v\n", dest)
+			// create a symbolic link from source to parent
+			err = os.Symlink(parent, dest)
+			if err != nil {
+				return fmt.Errorf("failed to create symbolic link: %v", err)
+			}
+
+			return nil
+		}
+	}
+
 	err = os.Link(parent, dest)
 	if err != nil {
 		err = copyFile(ctx, source, dest)