CI: add automatic docker container builds

Also: - fix file compression in build workflow - was using xz extension with gzip format - made the installer script a little more robust. It handles gzip assets now - Add some workflow badges to the readme
QutEcoacoustics · Jun 27, 2021 · d41ffab · d41ffab
1 parent 644fa0d
commit d41ffab
Show file tree

Hide file tree

Showing 7 changed files with 114 additions and 41 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -121,9 +121,11 @@ jobs:
 
           Set-CiOutput "AP_VERSION" "${env:AP_Version}"
           $extension = $IsWindows ? ".zip" : ".tar.xz"
+          $configuration_tag =  '${{ matrix.configuration }}' -eq 'Release' ?  '' : '_Debug'
           $artifact_tag = "AP_${variant_tag}_${{ matrix.configuration }}_${env:AP_Version}"
           Set-CiOutput "ARTIFACT_TAG" "$artifact_tag"
-          $artifact_path = "${{ runner.temp }}/AP_${variant_tag}_${{ matrix.configuration }}_${env:AP_Version}${extension}"
+
+          $artifact_path = "${{ runner.temp }}/AP_${variant_tag}${configuration_tag}_v${env:AP_Version}${extension}"
           Set-CiOutput "ARTIFACT_PATH" "$artifact_path"
 
       - uses: actions/cache@v2
@@ -197,10 +199,10 @@ jobs:
         with:
           macos: |
             cd "${{runner.temp}}/publish"
-            tar -cvzf "${{ steps.calc_vars.outputs.ARTIFACT_PATH }}" *
+            tar -cvJf "${{ steps.calc_vars.outputs.ARTIFACT_PATH }}" *
           linux: |
             cd "${{runner.temp}}/publish"
-            tar -cvzf "${{ steps.calc_vars.outputs.ARTIFACT_PATH }}" *
+            tar -cvJf "${{ steps.calc_vars.outputs.ARTIFACT_PATH }}" *
           windows: >
             7z a -tzip "${{ steps.calc_vars.outputs.ARTIFACT_PATH }}" "${{runner.temp}}\publish\*"
 

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -0,0 +1,39 @@
+name: publish docker
+
+on:
+  release:
+    types: [published]
+  # Manual trigger of a release
+  workflow_dispatch:
+    inputs:
+      name:
+        description: "Reason"
+        required: true
+        default: ""
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          lfs: false
+
+      - name: Set up Docker Buildx
+        uses: docker/[email protected]
+
+      - name: Login to DockerHub
+        uses: docker/[email protected]
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Run the build script
+        shell: pwsh
+        run: |
+          $version = gh release view --json tagName,targetCommitish | ConvertFrom-Json
+          $version | Write-Output
+          ./build/build_and_push -version $version.tagName -commit $version.targetCommitish
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/README.md b/README.md
@@ -5,8 +5,7 @@ Documentation (_in progress_) can be found at <https://ap.qut.ecoacoustics.info/
 
 ## Quick links
 
- - Come chat on [Gitter](https://gitter.im/QutEcoacoustics/audio-analysis) with us 🙂  
-   [![Gitter](https://badges.gitter.im/QutEcoacoustics/audio-analysis.svg)](https://gitter.im/QutEcoacoustics/audio-analysis?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
+ - Ask questions or start a discussion with us in the [discussions](https://github.com/QutEcoacoustics/audio-analysis/discussions) 🙂  
  - See the **[docs](https://ap.qut.ecoacoustics.info/)** for instructions on
    - Downloading AnalysisPrograms.exe
    - Running the program
@@ -47,12 +46,15 @@ citations will printed in the console and in the log file.
 
 ## Build status
 
-[![Build Status](https://dev.azure.com/QutEcoacoustics/audio-analysis/_apis/build/status/QutEcoacoustics.audio-analysis?branchName=master)](https://dev.azure.com/QutEcoacoustics/audio-analysis/_build/latest?definitionId=3&branchName=master)
+[![build](https://github.com/QutEcoacoustics/audio-analysis/actions/workflows/build.yml/badge.svg)](https://github.com/QutEcoacoustics/audio-analysis/actions/workflows/build.yml)
+[![docs](https://github.com/QutEcoacoustics/audio-analysis/actions/workflows/docs.yml/badge.svg)](https://github.com/QutEcoacoustics/audio-analysis/actions/workflows/docs.yml)
+[![docs](https://github.com/QutEcoacoustics/audio-analysis/actions/workflows/docs.yml/badge.svg)](https://github.com/QutEcoacoustics/audio-analysis/actions/workflows/docs.yml)
+[![release](https://github.com/QutEcoacoustics/audio-analysis/actions/workflows/release.yml/badge.svg)](https://github.com/QutEcoacoustics/audio-analysis/actions/workflows/release.yml)
 
 Weekly versions of AnalysisPrograms.exe are now built automatically at midnight Monday night.
 You can get copies from the [Releases](https://github.com/QutBioacoustics/audio-analysis/releases) page.
 
-Per-commit (the very latest) builds can be found from the Azure Pipelines page.
+Per-commit (the very latest) builds can be found from the [Actions](https://github.com/QutEcoacoustics/audio-analysis/actions) page.
 
 # License
 

diff --git a/build/Dockerfile b/build/Dockerfile
@@ -1,21 +1,23 @@
-FROM  debian:stretch-slim
-
-ARG GIT_COMMIT
-ARG AP_SOURCE="github"
-ARG AP_VERSION="latest"
-
-LABEL maintainer="Anthony Truskinger <[email protected]>" \
-      description="Debian environment for running AnalysisPrograms.exe" \
-      version="1.0" \
-      org.ecosounds.name="AnalysisPrograms.exe" \
-      org.ecosounds.version=${AP_VERSION} \
-      org.ecosounds.vendor="QUT Ecoacoustics" \
-      org.ecosounds.url="https://github.com/QutEcoacoustics/audio-analysis" \
-      org.ecosounds.vcs-url="https://github.com/QutEcoacoustics/audio-analysis" \
-      org.ecosounds.vcs-ref=${GIT_COMMIT} \
-      org.ecosounds.schema-version="1.0"
+# syntax=docker/dockerfile:1.2
 
+FROM  debian:stretch-slim
 
+ARG GIT_COMMIT=""
+ARG AP_VERSION=""
+ARG CREATION_DATE=""
+
+LABEL \
+    info.ecoacoustics.qut.ap.created=${CREATION_DATE} \
+    info.ecoacoustics.qut.ap.authors="Anthony Truskinger <[email protected]>" \
+    info.ecoacoustics.qut.ap.url="https://github.com/QutEcoacoustics/audio-analysis" \
+    info.ecoacoustics.qut.ap.documentation="https://ap.qut.ecoacoustics.info/" \
+    info.ecoacoustics.qut.ap.source="https://github.com/QutEcoacoustics/audio-analysis/blob/master/build/Dockerfile" \
+    info.ecoacoustics.qut.ap.version="${AP_VERSION}" \
+    info.ecoacoustics.qut.ap.revision=${GIT_COMMIT} \
+    info.ecoacoustics.qut.ap.vendor="QUT Ecoacoustics" \
+    info.ecoacoustics.qut.ap.licenses="Apache-2.0" \
+    info.ecoacoustics.qut.ap.title="AnalysisPrograms.exe" \
+    info.ecoacoustics.qut.ap.description="Debian environment for running AnalysisPrograms.exe"
 
 # Install system components (used by powershell. and AP as well)
 RUN apt-get update && apt-get install -y curl gnupg apt-transport-https unzip \
@@ -33,7 +35,7 @@ RUN add-apt-repository "deb http://mp3splt.sourceforge.net/repository unstable m
 
 # Powershell
 RUN \
-     # Import the public repository GPG keys
+    # Import the public repository GPG keys
     curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \
     # Register the Microsoft Product feed
     && echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-debian-stretch-prod stretch main" > /etc/apt/sources.list.d/microsoft.list \
@@ -44,6 +46,8 @@ RUN \
 
 # Install AP.exe
 ADD download_ap.ps1 /download_ap.ps1
-RUN /usr/bin/pwsh -NonInteractive -c "/download_ap.ps1 ${AP_SOURCE} -version ${AP_VERSION}"
+RUN --mount=type=secret,id=GITHUB_AUTH_TOKEN  \
+    /usr/bin/pwsh -NonInteractive -c \
+    '/download_ap.ps1 -Install -version "${AP_VERSION}" -GithubApiToken "$(Get-Content -Raw /run/secrets/GITHUB_AUTH_TOKEN)"'
 
 ENTRYPOINT [ "/bin/bash" ]
diff --git a/build/build_docker.ps1 b/build/build_docker.ps1
@@ -1,27 +1,39 @@
 #!/usr/bin/pwsh
 
 # builds and pushes a docker file to docker hub
-# currently only builds the 'stable' tag, which is applied to whatever version
-# is supplied to this script.
-# Future work: allow this script to build our 'Weekly' and 'Continuous' lines
-# as their own containers.
 param(
     # Version tag
-    [Parameter(Mandatory=$true)]
+    [Parameter(Mandatory = $true)]
     [string]
-    $version 
+    $version,
+
+    $commit
 )
+. $PSScriptRoot/log.ps1
+
+log "Resolving commit '$commit'"
 
 # get the current git commit
-$GIT_COMMIT=git log -1 --format=%H
+# this will resolve a name (like master/main) into a hash id
+# and will return current commit id if empty reference provided
+$commit = git log -1 --format=%H "$commit"
+
+log "Commit resolved to '$commit'"
+
+$env:DOCKER_BUILDKIT=1
+
+log "Building container" "Building"
 
 docker build `
     -t qutecoacoustics/audio-analysis:stable `
     -t qutecoacoustics/audio-analysis:latest `
     -t qutecoacoustics/audio-analysis:$version `
     . `
-    --build-arg GIT_COMMIT=$GIT_COMMIT `
-    --build-arg AP_SOURCE="github" `
-    --build-arg AP_VERSION=$version
+    --build-arg GIT_COMMIT=$commit `
+    --build-arg AP_VERSION=$version `
+    --build-arg CREATION_DATE=((Get-Date ).ToString("O"))
+    --secret id=GITHUB_AUTH_TOKEN,env=GITHUB_AUTH_TOKEN
+
+log "Pushing container" "Pushing"
 
 docker push qutecoacoustics/audio-analysis:stable
diff --git a/build/download_ap.ps1 b/build/download_ap.ps1
@@ -157,6 +157,7 @@ $headers = if ($GithubApiToken) {
         Write-Debug "Not using a github auth token"
         @{}
     }
+$headers['Accept'] = 'application/json'
 
 $base_indent = "- "
 $indent = "  - "
@@ -234,15 +235,20 @@ function Get-AssetUrl($prerelease ) {
             throw "Unknown platform. We could not detect your platform or we do not have a build for your platform."
         }
         Write-Debug "${indent}Querying GitHub with $github_url"
-        $response = Invoke-RestMethod -Method Get -Uri $github_url -Headers $headers
-        $response = $response | Select-Object -First 1
-        $asset_url = $response.assets `
+        $response = Invoke-RestMethod -Method Get -Uri $github_url -Headers $headers | % assets
+        $assets = $response
+
+        $asset_url = $assets `
             | Where-Object { $_.name -like "*$build*" } `
             | ForEach-Object browser_download_url
 
+        Write-Debug "${indent}Asset url: $asset_url. Available assets: $($response.assets.Count)"
         if($asset_url.Count -gt 1)  {
             throw "More than one asset matched criteria, need one url to continue. Urls: $asset_url"
         }
+        if ($null -eq $assert_url.Count) {
+            throw "Could not determine which asset to download. Do we support your OS and CPU architecture?"
+        }
 
         $final_version = $response.tag_name -replace "^v",""
         $is_prerelease = $response.prerelease
@@ -282,7 +288,8 @@ function Get-Asset($asset_url) {
     $extension = switch ($asset_url) {
         { $_.EndsWith(".zip") } { ".zip" }
         { $_.EndsWith(".tar.xz") } { ".tar.xz" }
-        Default { "Unspported extension for $asset_url"}
+        { $_.EndsWith(".tar.gz") } { ".tar.gz" }
+        Default { throw "Unsupported extension for $asset_url"}
     }
     $downloaded_zip = "$Destination${dir_seperator}AP$extension"
     Write-Debug "${indent}Downloading $asset_url to $downloaded_zip"
@@ -310,6 +317,13 @@ function Get-Asset($asset_url) {
                 throw "Failed extracting $downloaded_zip using tar"
             }
         }
+        elseif ($extension -eq ".tar.gz") {
+            # we should be only mac/linux at this branch
+            tar --extract --gzip --file $downloaded_zip --directory $Destination
+            if ($LASTEXITCODE -ne 0) {
+                throw "Failed extracting $downloaded_zip using tar"
+            }
+        }
         elseif (Get-Command unzip -CommandType Application -ErrorAction SilentlyContinue) {
             unzip -q -o $downloaded_zip -d $Destination
 
@@ -432,7 +446,7 @@ $actions =  [ordered]@{
             if ($LASTEXITCODE -ne 0) { return $false } else { return $true }
         } )
         "Check" =  ("AP executable should have execute permission", {
-            if($IsWindows) { return $null } 
+            if($IsWindows) { return $null }
             $(test -x $ap_path ; 0 -eq $LASTEXITCODE)
         } )
         "Uninstall" = $null

diff --git a/build/release_notes.ps1 b/build/release_notes.ps1
@@ -61,7 +61,7 @@ foreach ($line in $commit_summary) {
     | Select-Object -ExpandProperty Matches `
     | Select-Object -ExpandProperty Value `
     | Where-Object { '' -ne $_ }
-    $issue_refs  | Write-Debug
+
     $current.Issues += $issue_refs
 
 }