Merge pull request #1 from globekeeper/daniel/upstream-v1.3.4

Daniel/upstream v1.3.4

.dockerignore

@@ -1,3 +1,4 @@
+*.key
 /webui
 /.idea
 /bin
@@ -7,9 +8,18 @@
 /config
 /gdpr-data
 /ipfs
+/dev/conduit-db
+/dev/psql
+/vcpkg
+/libheif
+
+# Generated files
+assets.bin.go
 
 media-repo*.yaml
 homeserver.yaml
+s3-probably-safe-to-delete.txt
+/test.test
 
 # Binaries for programs and plugins
 *.exe

.github/workflows/binaries.yml

@@ -0,0 +1,80 @@
+name: Build release binaries
+
+on:
+  push:
+    #branches: ['travis/release-bin']
+    tags: ["*"]
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+          - windows-latest
+        arch:
+          - amd64
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.21'
+
+      - name: Install libheif (linux)
+        if: runner.os == 'linux'
+        run: "chmod +x ./.github/workflows/build-libheif.sh && ./.github/workflows/build-libheif.sh"
+
+      - name: Install libheif (windows)
+        if: runner.os == 'windows'
+        run: |
+          choco install pkgconfiglite
+          git clone https://github.com/Microsoft/vcpkg.git
+          cd vcpkg
+          .\bootstrap-vcpkg.bat
+          .\vcpkg integrate install
+          .\vcpkg install libde265:x64-windows
+          .\vcpkg install libheif:x64-windows
+          cd ..
+
+      - name: Dist
+        shell: bash
+        run: |
+          set -ex
+          pkgconfig=$PWD/vcpkg/installed/x64-windows/lib/pkgconfig
+          platform=${{ runner.os }}
+          platform=${platform,,}
+          mkdir -p $PWD/bin/dist
+          GOBIN=$PWD/bin go install -v ./cmd/utilities/compile_assets
+          $PWD/bin/compile_assets
+          pth="$platform-${{ matrix.arch }}"
+          mkdir -p $PWD/bin/$pth
+          PKG_CONFIG_PATH=$pkgconfig GOOS=$platform GOARCH=${{ matrix.arch }} GOBIN=$PWD/bin go build -o $PWD/bin/$pth -a -ldflags "-X github.com/t2bot/matrix-media-repo/common/version.Version=$(git describe --tags)" -v ./cmd/...
+          PKG_CONFIG_PATH=$pkgconfig GOOS=$platform GOARCH=${{ matrix.arch }} GOBIN=$PWD/bin go build -pgo=pgo_media_repo.pprof -o $PWD/bin/$pth -a -ldflags "-X github.com/t2bot/matrix-media-repo/common/version.Version=$(git describe --tags)" -v ./cmd/workers/media_repo
+          cd $PWD/bin/$pth
+          arch=${{ matrix.arch }}
+          if [ "$arch" == "amd64" ]; then
+            arch="x64"
+          fi
+          if [ "$platform" == "windows" ]; then
+            for file in * ; do mv -v $file ../dist/${file%.*}-win-${arch}.exe; done;
+          else
+            for file in * ; do mv -v $file ../dist/${file}-${platform}-${arch}; done;
+          fi
+          cd ../../
+          rm -rv $PWD/bin/dist/compile_assets*
+
+      - name: Upload
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.os }}-${{ matrix.arch }}
+          path: ./bin/dist/*

.github/workflows/build-libheif.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -ex
+sudo apt-get install -y git cmake make pkg-config libx265-dev libde265-dev libjpeg-dev libtool
+git clone https://github.com/strukturag/libheif.git
+cd libheif
+git checkout v1.17.6
+mkdir build
+cd build
+cmake --preset=release ..
+make
+sudo make install
+sudo ldconfig

.github/workflows/docker.yml

@@ -0,0 +1,113 @@
+name: Create and publish a Docker image
+
+on:
+  push:
+    branches: ['main']
+    tags: ['*']
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm/v6
+          - linux/arm/v7
+          - linux/arm64
+
+    steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: ${{ matrix.platform }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.REGISTRY  }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true
+
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"      
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ env.PLATFORM_PAIR }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    runs-on: ubuntu-latest
+    needs: [build]
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY  }}/${{ env.IMAGE_NAME }}
+
+      - name: Login
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+          $(printf '${{ env.REGISTRY  }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
+
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.REGISTRY  }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} 

.github/workflows/download_pgo.yml

@@ -0,0 +1,19 @@
+name: "Update PGO performance profile"
+on:
+#  schedule:
+#    - cron: "0 0 * * 2" # Every Tuesday at 00:00
+  workflow_dispatch:
+jobs:
+  update_pgo:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    env:
+      PGO_MERGE: ${{ secrets.PGO_MERGE }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: "Download new pgo_media_repo.pprof"
+        run: "curl -sv --fail -X POST -H \"Authorization: Bearer ${PGO_MERGE}\" https://pgo-mmr.t2host.io/v1/merge?and_combine=true > pgo_media_repo.pprof"
+      - uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: "Update pgo_media_repo.pprof"

.github/workflows/download_providers.yml

@@ -0,0 +1,17 @@
+name: "Update oEmbed providers"
+on:
+#  schedule:
+#    - cron: "0 0 * * 2" # Every Tuesday at 00:00
+  workflow_dispatch:
+jobs:
+  update_providers:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: "Download new providers.json"
+        run: "curl -s --fail https://oembed.com/providers.json > assets/providers.json"
+      - uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: "Update providers.json"

.github/workflows/main.yml

@@ -3,32 +3,46 @@ on:
   push:
 jobs:
   build:
-    name: 'Build Go 1.16'
+    name: 'Go Build'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.16'
-      - run: './build.sh'
-  complement:
-    name: 'Complement'
+          go-version: '1.21'
+      - name: "Install libheif"
+        run: "chmod +x ./.github/workflows/build-libheif.sh && ./.github/workflows/build-libheif.sh"
+      - run: './build.sh' # verify the thing compiles
+  static:
+    name: 'Go Static'
     runs-on: ubuntu-latest
-    container:
-      # https://github.com/matrix-org/complement/blob/master/dockerfiles/ComplementCIBuildkite.Dockerfile
-      image: matrixdotorg/complement:latest
-      env:
-        CI: true
-      ports:
-        - 8448:8448
-      volumes:
-        - /var/run/docker.sock:/var/run/docker.sock
     steps:
-      - uses: actions/checkout@v2
-      - run: docker build -t complement-media-repo -f Complement.Dockerfile .
-      - run: chmod +x ci-complement.sh
-      - run: ./ci-complement.sh
-        env:
-          CI: "true"
-          COMPLEMENT_BASE_IMAGE: "complement-media-repo"
-          COMPLEMENT_SPAWN_HS_TIMEOUT_SECS: "120"
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.21'
+      - name: "Install libheif"
+        run: "chmod +x ./.github/workflows/build-libheif.sh && ./.github/workflows/build-libheif.sh"
+      - name: "Prepare: compile assets"
+        run: "GOBIN=$PWD/bin go install -v ./cmd/utilities/compile_assets"
+      - name: "Run: compile assets"
+        run: "$PWD/bin/compile_assets"
+      - name: "Prepare: staticcheck"
+        run: 'go install honnef.co/go/tools/cmd/staticcheck@latest'
+      - run: 'go vet ./cmd/...'
+      - run: 'staticcheck ./cmd/...'
+  test:
+    name: 'Go Test'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.21'
+      - name: "Prepare: compile assets"
+        run: "GOBIN=$PWD/bin go install -v ./cmd/utilities/compile_assets"
+      - name: "Run: compile assets"
+        run: "$PWD/bin/compile_assets"
+      - name: "Run: tests"
+        run: "go test -c -v ./test && ./test.test '-test.v'" # cheat and work around working directory issues
+        timeout-minutes: 30

.gitignore

@@ -1,3 +1,4 @@
+*.key
 /webui
 /.idea
 /bin
@@ -9,13 +10,16 @@
 /ipfs
 /dev/conduit-db
 /dev/psql
+/vcpkg
+/libheif
 
 # Generated files
 assets.bin.go
 
 media-repo*.yaml
 homeserver.yaml
 s3-probably-safe-to-delete.txt
+/test.test
 
 # Binaries for programs and plugins
 *.exe