Fix Identify default git branch when downloading pipelines (nextflow-…

…io#3593) (nextflow-io#5375)


Signed-off-by: Tom Sellman <[email protected]>
Signed-off-by: Paolo Di Tommaso <[email protected]>
Co-authored-by: Paolo Di Tommaso <[email protected]>

modules/nextflow/src/main/groovy/nextflow/config/Manifest.groovy

@@ -19,7 +19,6 @@ package nextflow.config
 
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
-import static nextflow.Const.DEFAULT_BRANCH
 import static nextflow.Const.DEFAULT_MAIN_FILE_NAME
 /**
  * Models the nextflow config manifest settings
@@ -52,7 +51,7 @@ class Manifest {
 
 
     String getDefaultBranch() {
-        target.defaultBranch ?: DEFAULT_BRANCH
+        target.defaultBranch
     }
 
     String getDescription() {

modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy

@@ -55,6 +55,8 @@ import org.eclipse.jgit.merge.MergeStrategy
 @Slf4j
 @CompileStatic
 class AssetManager {
+    private static final String REMOTE_REFS_ROOT = "refs/remotes/origin/"
+    private static final String REMOTE_DEFAULT_HEAD = REMOTE_REFS_ROOT + "HEAD"
 
     /**
      * The folder all pipelines scripts are installed
@@ -422,7 +424,16 @@ class AssetManager {
     }
 
     String getDefaultBranch() {
-        getManifest().getDefaultBranch()
+        // if specified in manifest, that takes priority
+        // otherwise look for a symbolic ref (refs/remotes/origin/HEAD)
+        return getManifest().getDefaultBranch()
+                ?: getRemoteBranch()
+                ?: DEFAULT_BRANCH
+    }
+
+    protected String getRemoteBranch() {
+        Ref remoteHead = git.getRepository().findRef(REMOTE_DEFAULT_HEAD)
+        return remoteHead?.getTarget()?.getName()?.substring(REMOTE_REFS_ROOT.length())
     }
 
     @Memoized
@@ -582,7 +593,7 @@ class AssetManager {
             final cloneURL = getGitRepositoryUrl()
             log.debug "Pulling $project -- Using remote clone url: ${cloneURL}"
 
-            // clone it
+            // clone it, but don't specify a revision - jgit will checkout the default branch
             def clone = Git.cloneRepository()
             if( provider.hasCredentials() )
                 clone.setCredentialsProvider( provider.getGitCredentials() )
@@ -595,9 +606,25 @@ class AssetManager {
                 clone.setDepth(deep)
             clone.call()
 
+            // git cli would automatically create a 'refs/remotes/origin/HEAD' symbolic ref pointing at the remote's
+            // default branch. jgit doesn't do this, but since it automatically checked out the default branch on clone
+            // we can create the symbolic ref ourselves using the current head
+            def head = git.getRepository().findRef(Constants.HEAD)
+            if( head ) {
+                def headName = head.isSymbolic()
+                    ? Repository.shortenRefName(head.getTarget().getName())
+                    : head.getName()
+
+                git.repository.getRefDatabase()
+                    .newUpdate(REMOTE_DEFAULT_HEAD, true)
+                    .link(REMOTE_REFS_ROOT + headName)
+            } else {
+                log.debug "Unable to determine default branch of repo ${cloneURL}, symbolic ref not created"
+            }
+
+            // now the default branch is recorded in the repo, explicitly checkout the revision (if specified).
+            // this also allows 'revision' to be a SHA commit id, which isn't supported by the clone command
             if( revision ) {
-                // use an explicit checkout command *after* the clone instead of cloning a specific branch
-                // because the clone command does not allow the use of SHA commit id (only branch and tag names)
                 try { git.checkout() .setName(revision) .call() }
                 catch ( RefNotFoundException e ) { checkoutRemoteBranch(revision) }
             }
@@ -729,6 +756,9 @@ class AssetManager {
         }
     }
 
+    static boolean isRemoteBranch(Ref ref) {
+        return ref.name.startsWith(REMOTE_REFS_ROOT) && ref.name != REMOTE_DEFAULT_HEAD
+    }
 
     /**
      * @return A list of existing branches and tags names. For example
@@ -750,7 +780,7 @@ class AssetManager {
         def master = getDefaultBranch()
 
         List<String> branches = getBranchList()
-            .findAll { it.name.startsWith('refs/heads/') || it.name.startsWith('refs/remotes/origin/') }
+            .findAll { it.name.startsWith('refs/heads/') || isRemoteBranch(it) }
             .unique { shortenRefName(it.name) }
             .collect { Ref it -> refToString(it,current,master,false,level) }
 
@@ -785,7 +815,7 @@ class AssetManager {
 
         Map<String, Ref> remote = checkForUpdates ? git.lsRemote().callAsMap() : null
         getBranchList()
-                .findAll { it.name.startsWith('refs/heads/') || it.name.startsWith('refs/remotes/origin/') }
+                .findAll { it.name.startsWith('refs/heads/') || isRemoteBranch(it) }
                 .unique { shortenRefName(it.name) }
                 .each { Ref it -> branches << refToMap(it,remote)  }
 
@@ -882,7 +912,7 @@ class AssetManager {
 
         def remote = git.lsRemote().callAsMap()
         List<String> branches = getBranchList()
-                .findAll { it.name.startsWith('refs/heads/') || it.name.startsWith('refs/remotes/origin/') }
+                .findAll { it.name.startsWith('refs/heads/') || isRemoteBranch(it) }
                 .unique { shortenRefName(it.name) }
                 .findAll { Ref ref -> hasRemoteChange(ref,remote) }
                 .collect { Ref ref -> formatUpdate(remote.get(ref.name),level) }

modules/nextflow/src/test/groovy/nextflow/cli/CmdInfoTest.groovy

@@ -71,6 +71,7 @@ class CmdInfoTest extends Specification {
         screen.contains(" main script : main.nf")
         screen.contains(" revisions   : ")
         screen.contains(" * master (default)")
+        !screen.contains("   HEAD")
     }
 
     def 'should print json info' () {
@@ -89,11 +90,12 @@ class CmdInfoTest extends Specification {
         json.repository == "https://github.com/nextflow-io/hello"
         json.localPath == "$tempDir/nextflow-io/hello"
         json.manifest.mainScript == 'main.nf'
-        json.manifest.defaultBranch == 'master'
+        json.manifest.defaultBranch == null
         json.revisions.current == 'master'
         json.revisions.master == 'master'
         json.revisions.branches.size()>1
         json.revisions.branches.any { it.name == 'master' }
+        !json.revisions.branches.any { it.name == 'HEAD' }
         json.revisions.tags.size()>1
         json.revisions.tags.any { it.name == 'v1.1' }
 
@@ -115,11 +117,12 @@ class CmdInfoTest extends Specification {
         json.repository == "https://github.com/nextflow-io/hello"
         json.localPath == "$tempDir/nextflow-io/hello"
         json.manifest.mainScript == 'main.nf'
-        json.manifest.defaultBranch == 'master'
+        json.manifest.defaultBranch == null
         json.revisions.current == 'master'
         json.revisions.master == 'master'
         json.revisions.branches.size()>1
         json.revisions.branches.any { it.name == 'master' }
+        !json.revisions.branches.any { it.name == 'HEAD' }
         json.revisions.tags.size()>1
         json.revisions.tags.any { it.name == 'v1.1' }
 

modules/nextflow/src/test/groovy/nextflow/scm/AssetManagerTest.groovy

@@ -414,7 +414,7 @@ class AssetManagerTest extends Specification {
         then:
         holder.getMainScriptName() == 'main.nf'
         holder.getHomePage() == 'https://github.com/foo/bar'
-        holder.manifest.getDefaultBranch() == 'master'
+        holder.manifest.getDefaultBranch() == null
         holder.manifest.getDescription() == null
 
     }
@@ -601,4 +601,54 @@ class AssetManagerTest extends Specification {
         noExceptionThrown()
     }
 
+    @Requires({System.getenv('NXF_GITHUB_ACCESS_TOKEN')})
+    def 'should identify default branch when downloading repo'() {
+
+        given:
+        def folder = tempDir.getRoot()
+        def token = System.getenv('NXF_GITHUB_ACCESS_TOKEN')
+        def manager = new AssetManager().build('nextflow-io/socks', [providers: [github: [auth: token]]])
+
+        when:
+        // simulate calling `nextflow run nextflow-io/socks` without specifying a revision
+        manager.download()
+        manager.checkout(null)
+        then:
+        folder.resolve('nextflow-io/socks/.git').isDirectory()
+        manager.getCurrentRevision() == 'main'
+
+        when:
+        manager.download()
+        then:
+        noExceptionThrown()
+    }
+
+    @Requires({System.getenv('NXF_GITHUB_ACCESS_TOKEN')})
+    def 'can filter remote branches'() {
+        given:
+        def folder = tempDir.getRoot()
+        def token = System.getenv('NXF_GITHUB_ACCESS_TOKEN')
+        def manager = new AssetManager().build('nextflow-io/hello', [providers: [github: [auth: token]]])
+        manager.download()
+        def branches = manager.getBranchList()
+
+        when:
+        def remote_head = branches.find { it.name == 'refs/remotes/origin/HEAD' }
+        then:
+        remote_head != null
+        !AssetManager.isRemoteBranch(remote_head)
+
+        when:
+        def remote_master = branches.find { it.name == 'refs/remotes/origin/master' }
+        then:
+        remote_master != null
+        AssetManager.isRemoteBranch(remote_master)
+
+        when:
+        def local_master = branches.find { it.name == 'refs/heads/master' }
+        then:
+        local_master != null
+        !AssetManager.isRemoteBranch(local_master)
+    }
+
 }