Release 0.2.0 (#87)

Enhancements: * Runner is now written in Go * Cache docker layers in GitHub Actions builds * CI/CD embeds version info using -ldflags * Controller and Runner now have --help and --version flags Bug fixes: * Running cards don't show the video encoding symbol (#75) * The Controller still deletes the original file even if the Runner reports that it failed (#81) Documentation: * Added Go version badge for Runner * Added Cross-platform as a reason for choosing Encodarr * Added TZ env var to Docker and Docker Compose commands * Rewrite Startup Configuration section for the Runner to match the options available with the Go Runner * Remove Go Runner from Future Plans section
BrenekH · Apr 23, 2021 · d48f150 · d48f150
2 parents b9ecef9 + 86ac416
commit d48f150
Show file tree

Hide file tree

Showing 53 changed files with 2,940 additions and 701 deletions.
diff --git a/.github/workflows/controller.yaml b/.github/workflows/controller.yaml
@@ -66,14 +66,23 @@ jobs:
         if: "!contains(matrix.go-os-arch, 'windows')"
         run: echo "EXEC_SUFFIX=$("")" >> $GITHUB_ENV
 
+      - name: Set version for ldflags (tag ref)
+        if: startsWith(github.ref, 'refs/tags/')
+        run: echo "LDFLAGS_VERSION=$(echo ${GITHUB_REF:10})" >> $GITHUB_ENV
+
+      - name: Set version for ldflags (non-tag ref)
+        if: "!startsWith(github.ref, 'refs/tags/')"
+        # Makes the embedded version "{branch-name}-development"
+        run: echo "LDFLAGS_VERSION=$(echo ${GITHUB_REF:11})-development" >> $GITHUB_ENV
+
       - name: Build executable
         env:
           # We aim to be CGO free to improve compatibility. Disabling it in the CI should help with enforcing that goal.
           CGO_ENABLED: 0
           GOARM: 7
           GOOS: ${{ env.COMP_GOOS }}
           GOARCH: ${{ env.COMP_GOARCH }}
-        run: go build -o encodarr-controller-${{ env.COMP_GOOS }}-${{ env.COMP_GOARCH }}${{ env.EXEC_SUFFIX }}
+        run: go build -o encodarr-controller-${{ env.COMP_GOOS }}-${{ env.COMP_GOARCH }}${{ env.EXEC_SUFFIX }} -ldflags="-X 'github.com/BrenekH/encodarr/controller/options.Version=${{ env.LDFLAGS_VERSION }}'"
 
       - name: Upload artifact
         uses: actions/upload-artifact@v2
@@ -108,6 +117,14 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-controller-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-controller-buildx-
+
       - name: Login to DockerHub
         uses: docker/login-action@v1
         with:
@@ -121,14 +138,34 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Set version for ldflags (tag ref)
+        if: startsWith(github.ref, 'refs/tags/')
+        run: echo "LDFLAGS_VERSION=$(echo ${GITHUB_REF:10})" >> $GITHUB_ENV
+
+      - name: Set version for ldflags (non-tag ref)
+        if: "!startsWith(github.ref, 'refs/tags/')"
+        # Makes the embedded version "{branch-name}-development"
+        run: echo "LDFLAGS_VERSION=$(echo ${GITHUB_REF:11})-development" >> $GITHUB_ENV
+
       - name: Build and push container images/tags
         uses: docker/build-push-action@v2
         with:
           context: ./controller
           push: true
           platforms: linux/amd64,linux/arm64,linux/arm/v7
+          build-args: LDFLAGS_VERSION=${{ env.LDFLAGS_VERSION }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+      - name: Move cache
+        # Temp fix
+        # https://github.com/docker/build-push-action/issues/252
+        # https://github.com/moby/buildkit/issues/1896
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
 
   upload-binaries-to-gh-releases:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/runner.yaml b/.github/workflows/runner.yaml
@@ -8,8 +8,8 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ['3.6', '3.7', '3.8', '3.9']
+        os: ['ubuntu-latest']
+        go-version: ['1.16']
 
     defaults:
       run:
@@ -18,28 +18,27 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+      - name: Setup Go ${{ matrix.go-version }}
+        uses: actions/setup-go@v2
         with:
-          python-version: ${{ matrix.python-version }}
-          architecture: x64
+          go-version: ${{ matrix.go-version }}
 
-      - name: Install Python dependencies
-        run: python -m pip install -r requirements.txt
-
-      - name: Install Pytest
-        run: python -m pip install pytest
-
-      - name: Test using Pytest
-        run: python -m pytest
+      - name: Run tests
+        env:
+          # We aim to be CGO free to improve compatibility. Disabling it in the CI should help with that goal.
+          CGO_ENABLED: 0
+          ENCODARR_CONFIG_DIR: ${{ runner.temp }}
+          ENCODARR_TEMP_DIR: ${{ runner.temp }}
+        run: mkdir -p "${ENCODARR_TEMP_DIR}/Encodarr/Runner" && go test ./...
 
   build:
     runs-on: ${{ matrix.os }}
 
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ['3.9']
+        os: ['ubuntu-latest']
+        go-version: ['1.16']
+        go-os-arch: ['linux/amd64', 'linux/arm64', 'linux/arm', 'windows/amd64', 'darwin/amd64', 'darwin/arm64']
 
     defaults:
       run:
@@ -48,26 +47,49 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+      - name: Setup Go ${{ matrix.go-version }}
+        uses: actions/setup-go@v2
         with:
-          python-version: ${{ matrix.python-version }}
-          architecture: x64
-
-      - name: Install Python dependencies
-        run: python -m pip install -r requirements.txt
-
-      - name: Install PyInstaller
-        run: python -m pip install PyInstaller
-
-      - name: Create Executable
-        run: python -m PyInstaller --name "encodarr-runner-${{ runner.os }}-amd64" --onefile main.py
+          go-version: ${{ matrix.go-version }}
+
+      - name: Generate Env Vars
+        env:
+          IFS: "/"
+        run: |
+          echo "COMP_GOOS=$(echo ${{ matrix.go-os-arch }} | cut -d/ -f1 -)" >> $GITHUB_ENV
+          echo "COMP_GOARCH=$(echo ${{ matrix.go-os-arch }} | cut -d/ -f2 -)" >> $GITHUB_ENV
+
+      - name: Set suffix (windows)
+        if: contains(matrix.go-os-arch, 'windows')
+        run: echo "EXEC_SUFFIX=.exe" >> $GITHUB_ENV
+
+      - name: Set suffix (literally everything else)
+        if: "!contains(matrix.go-os-arch, 'windows')"
+        run: echo "EXEC_SUFFIX=$("")" >> $GITHUB_ENV
+
+      - name: Set version for ldflags (tag ref)
+        if: startsWith(github.ref, 'refs/tags/')
+        run: echo "LDFLAGS_VERSION=$(echo ${GITHUB_REF:10})" >> $GITHUB_ENV
+
+      - name: Set version for ldflags (non-tag ref)
+        if: "!startsWith(github.ref, 'refs/tags/')"
+        # Makes the embedded version "{branch-name}-development"
+        run: echo "LDFLAGS_VERSION=$(echo ${GITHUB_REF:11})-development" >> $GITHUB_ENV
+
+      - name: Build executable
+        env:
+          # We aim to be CGO free to improve compatibility. Disabling it in the CI should help with enforcing that goal.
+          CGO_ENABLED: 0
+          GOARM: 7
+          GOOS: ${{ env.COMP_GOOS }}
+          GOARCH: ${{ env.COMP_GOARCH }}
+        run: go build -o encodarr-runner-${{ env.COMP_GOOS }}-${{ env.COMP_GOARCH }}${{ env.EXEC_SUFFIX }} -ldflags="-X 'github.com/BrenekH/encodarr/runner/options.Version=${{ env.LDFLAGS_VERSION }}'" ./cmd/EncodarrRunner/main.go
 
       - name: Upload artifact
         uses: actions/upload-artifact@v2
         with:
-          name: encodarr-runner-${{ runner.os }}-amd64
-          path: ${{ github.workspace }}/runner/dist/*
+          name: encodarr-runner-${{ env.COMP_GOOS }}-${{ env.COMP_GOARCH }}${{ env.EXEC_SUFFIX }}
+          path: ${{ github.workspace }}/runner/encodarr-runner-${{ env.COMP_GOOS }}-${{ env.COMP_GOARCH }}${{ env.EXEC_SUFFIX }}
 
   deploy-container-images-tags:
     runs-on: ubuntu-latest
@@ -96,6 +118,14 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-runner-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-runner-buildx-
+
       - name: Login to DockerHub
         uses: docker/login-action@v1
         with:
@@ -109,14 +139,34 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Set version for ldflags (tag ref)
+        if: startsWith(github.ref, 'refs/tags/')
+        run: echo "LDFLAGS_VERSION=$(echo ${GITHUB_REF:10})" >> $GITHUB_ENV
+
+      - name: Set version for ldflags (non-tag ref)
+        if: "!startsWith(github.ref, 'refs/tags/')"
+        # Makes the embedded version "{branch-name}-development"
+        run: echo "LDFLAGS_VERSION=$(echo ${GITHUB_REF:11})-development" >> $GITHUB_ENV
+
       - name: Build and push container images/tags
         uses: docker/build-push-action@v2
         with:
           context: ./runner
           push: true
           platforms: linux/amd64,linux/arm64,linux/arm/v7
+          build-args: LDFLAGS_VERSION=${{ env.LDFLAGS_VERSION }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+      - name: Move cache
+        # Temp fix
+        # https://github.com/docker/build-push-action/issues/252
+        # https://github.com/moby/buildkit/issues/1896
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
 
   upload-binaries-to-gh-releases:
     runs-on: ubuntu-latest

diff --git a/README.md b/README.md
@@ -8,22 +8,29 @@
 ![Docker Pulls](https://img.shields.io/docker/pulls/brenekh/encodarr-runner?label=runner%20docker%20pulls)
 ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/brenekh/encodarr-controller/latest?label=controller%20image%20size)
 ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/brenekh/encodarr-runner/latest?label=runner%20image%20size)
-![GitHub go.mod Go version (subdirectory of monorepo)](https://img.shields.io/github/go-mod/go-version/brenekh/encodarr?filename=controller%2Fgo.mod)
+![GitHub go.mod Go version (subdirectory of monorepo)](https://img.shields.io/github/go-mod/go-version/brenekh/encodarr?filename=controller%2Fgo.mod&label=Controller%20Go%20Version)
+![GitHub go.mod Go version (subdirectory of monorepo)](https://img.shields.io/github/go-mod/go-version/brenekh/encodarr?filename=runner%2Fgo.mod&label=Runner%20Go%20Version)
 [![Controller CI/CD](https://github.com/BrenekH/encodarr/actions/workflows/controller.yaml/badge.svg)](https://github.com/BrenekH/encodarr/actions/workflows/controller.yaml)
 [![Runner CI/CD](https://github.com/BrenekH/encodarr/actions/workflows/runner.yaml/badge.svg)](https://github.com/BrenekH/encodarr/actions/workflows/runner.yaml)
 
 ## What is Encodarr?
 
 Encodarr is a self-hosted web application that encodes video files to a target format using distributed computing.
 
+<!-- TODO: Add information on the architecture (high level). Stuff like many Runners connect to a single Controller. -->
+
 ## Why use Encodarr?
-<!-- TODO: Why use Encodarr? (other than easy to setup) -->
+<!-- TODO: Why use Encodarr? (other than easy to setup and cross-platform) -->
 
 ### Easy to Setup
 
 Encodarr bypasses the need to share media across the network by instead transmitting the file to be operated on to the Runners.
 This means that Encodarr is much easier to setup than other solutions.
 
+### Cross-platform
+
+Both the Controller and Runner can be cross-compiled for any system [supported by the Go toolchain](https://gist.github.com/asukakenji/f15ba7e588ac42795f421b48b8aede63) including Raspberry Pis and M1 Macs.
+
 ## Dependencies
 
 The container images come with the dependencies installed, so if you go that route, all you need is [Docker](https://docs.docker.com/get-docker/).
@@ -81,6 +88,7 @@ Docker run:
 docker run -d \
   --name Encodarr-Runner \
   -v <path to runner data>:/config:rw \
+  -e TZ=Europe/London
   -e "ENCODARR_RUNNER_NAME=Runner 1" \
   -e ENCODARR_RUNNER_CONTROLLER_IP=<Controller IP> \
   -e ENCODARR_RUNNER_CONTROLLER_PORT=8123 \
@@ -99,6 +107,7 @@ services:
     volumes:
       - <path to runner data>:/config:rw
     environment:
+      - TZ=Europe/London
       - ENCODARR_RUNNER_NAME=Runner 1
       - ENCODARR_RUNNER_CONTROLLER_IP=<Controller IP>
       - ENCODARR_RUNNER_CONTROLLER_PORT=8123
@@ -113,8 +122,7 @@ In addition, the paths to data that are mounted to `/config` in the container sh
 ### Startup Configuration
 
 Startup values configured either through environment variables, or command line arguments.
-All of the command line variants expect a value after a space (`--port 8123`) expect the Runner `--debug` flag.
-It is a boolean flag.
+All of the command line variants expect a value after a space (`--port 8123`).
 
 #### Controller
 
@@ -129,15 +137,21 @@ In a container, this is pre-set to `/config`.
 
 #### Runner
 
-`ENCODARR_DEBUG`, `--debug` enables outputting debug messages to the log.
-If the environment variable is set to `True`, then debug messages are turned on.
-(default: `False`)
+`ENCODARR_CONFIG_DIR`, `--config-dir` sets the directory that the configuration files are saved to.
+This includes the log file.
+In a container, this is pre-set to `/config`.
+(default: `<platform user config directory>/encodarr/runner-<time of runner startup>/config`)
 
-`ENCODARR_LOG_FILE` sets the location of the runner log file.
-(default: `/config/runner.log`)
+`ENCODARR_TEMP_DIR`, `--temp-dir` sets the directory that the media files are saved to when they are being worked on.
+If you want to protect your flash memory(SSDs and SD Cards), you can set this to be on a hard drive.
+(default: `<platform user temp directory>`)
+
+`ENCODARR_LOG_LEVEL`, `--log-level` sets the level of output from the logging system to both the log file and the terminal output.
+Possible values are: `trace`, `debug`, `info`, `warn` (or `warning`, they are identical), `error`, `critical`.
+(default: `info`)
 
 `ENCODARR_RUNNER_NAME`, `--name` sets the name to be shown in the Web UI when referring to this runner.
-(default: `Runner-<random 3 digit number>`)
+(default: `<machine hostname>-<random number>`)
 
 `ENCODARR_RUNNER_CONTROLLER_IP`, `--controller-ip` sets the IP for connecting to the Controller.
 (default: `localhost`)
@@ -160,8 +174,6 @@ This project holds all maintainers, contributors, and participants to the standa
 
 ## Future Plans
 
-* Rewrite Runner from Python to Go (following Clean Architecture guidelines)
-
 * Rewrite Controller to conform to Clean Architecture guidelines
 
 * Instead of configuring with dropdowns and checkboxes, use a plugin system

diff --git a/controller/Dockerfile b/controller/Dockerfile
@@ -3,13 +3,14 @@ FROM golang:1.16-buster AS builder
 
 ARG TARGETOS
 ARG TARGETARCH
+ARG LDFLAGS_VERSION=development
 
 WORKDIR /go/src/encodarr/controller
 
 COPY . .
 
-# Disable CGO so that we have a static binary
-RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o encodarr .
+# Disable CGO so that we have a static binary, set the platform for multi-arch builds, and embed the Version.
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o encodarr -ldflags="-X 'github.com/BrenekH/encodarr/controller/options.Version=${LDFLAGS_VERSION}'" .
 
 
 # Run stage

diff --git a/controller/config/config.go b/controller/config/config.go
@@ -9,9 +9,6 @@ import (
 	"github.com/BrenekH/encodarr/controller/options"
 )
 
-// Version represents the current version of the Encodarr Controller
-const Version string = "0.0.1-alpha.1"
-
 // Settings is used to represent how settings are saved to a file
 type Settings struct {
 	HealthCheckInterval uint64

diff --git a/controller/controller/completed_looper.go b/controller/controller/completed_looper.go
@@ -69,6 +69,15 @@ func completedHandler(r JobCompleteRequest, wg *sync.WaitGroup) {
 		return
 	}
 
+	if r.Failed {
+		// Save the History and exit because we don't want to delete the original file when we don't have a replacement
+		err = r.History.Save()
+		if err != nil {
+			logger.Error(fmt.Sprintf("Error saving history: %v", err.Error()))
+		}
+		return
+	}
+
 	filename := dJob.Job.Path
 
 	if config.Global.SmallerFiles {