WIP: Refactor server database

.appveyor.yml

@@ -12,7 +12,7 @@ environment:
   global:
     MUMBLE_BUILD_NUMBER_TOKEN:
       secure: scRizsdwbTWjY7M9RHPT9EXDBxtWvXTsyVq1xXTWF1A=
-    MUMBLE_ENVIRONMENT_SOURCE: 'https://dl.mumble.info/build/vcpkg'
+    MUMBLE_ENVIRONMENT_SOURCE: 'https://github.com/mumble-voip/vcpkg/releases/download/2024-05/'
     MUMBLE_ENVIRONMENT_STORE: 'C:/MumbleBuild'
     MUMBLE_ENVIRONMENT_PATH: '%MUMBLE_ENVIRONMENT_STORE%/%MUMBLE_ENVIRONMENT_VERSION%'
     MUMBLE_ENVIRONMENT_TOOLCHAIN: '%MUMBLE_ENVIRONMENT_PATH%/scripts/buildsystems/vcpkg.cmake'
@@ -23,7 +23,7 @@ environment:
 
   matrix:
     - MUMBLE_ENVIRONMENT_TRIPLET: 'x64-windows-static-md-release'
-      MUMBLE_ENVIRONMENT_VERSION: 'mumble_env.x64-windows-static-md-release.2023-12-31.6a3ce9c65'
+      MUMBLE_ENVIRONMENT_VERSION: ' mumble_env.x64-windows-static-md-release.2024-05-19.5eaf0d45f'
 
 install:
   - ps: .ci/install-environment_windows.ps1

.ci/azure-pipelines/build_linux.bash

@@ -28,6 +28,7 @@ cd $BUILD_BINARIESDIRECTORY
 cmake -G Ninja -DCMAKE_INSTALL_PREFIX=appdir/usr -DCMAKE_UNITY_BUILD=ON \
       -DCMAKE_BUILD_TYPE=Release -DBUILD_NUMBER=$BUILD_NUMBER \
       -Dtests=ON -Dsymbols=ON \
+	  -Ddatabase-sqlite-tests=ON -Ddatabase-mysql-tests=ON -Ddatabase-postgresql-tests=ON \
       -Ddisplay-install-paths=ON $BUILD_SOURCESDIRECTORY
 
 cmake --build .

.ci/azure-pipelines/build_macos.bash

@@ -36,10 +36,13 @@ BUILD_NUMBER=$("./scripts/mumble-build-number.py" --commit "${BUILD_SOURCEVERSIO
 
 cd $BUILD_BINARIESDIRECTORY
 
+# Note: We can't use MySQL - see install script for details
 cmake -G Ninja -DVCPKG_TARGET_TRIPLET=$MUMBLE_ENVIRONMENT_TRIPLET -DCMAKE_TOOLCHAIN_FILE=$MUMBLE_ENVIRONMENT_TOOLCHAIN \
       -DIce_HOME="$MUMBLE_ENVIRONMENT_PATH/installed/$MUMBLE_ENVIRONMENT_TRIPLET" \
       -DCMAKE_BUILD_TYPE=Release -DCMAKE_UNITY_BUILD=ON -DBUILD_NUMBER=$BUILD_NUMBER \
       -Dtests=ON -Dstatic=ON -Dsymbols=ON \
+	  -Denable-mysql=OFF \
+	  -Ddatabase-sqlite-tests=ON -Ddatabase-postgresql-tests=ON \
       -Ddisplay-install-paths=ON $BUILD_SOURCESDIRECTORY
 
 cmake --build .

.ci/azure-pipelines/install-environment_linux.bash

@@ -35,3 +35,27 @@ sudo apt -y install \
 	zsync \
 	appstream \
 	libpoco-dev
+
+
+
+# Setup MySQL and PostgreSQL databases for the Mumble tests
+echo "Configuring MySQL..."
+
+echo -e "[mysqld]\nlog-bin-trust-function-creators = 1" | sudo tee -a /etc/mysql/my.cnf
+
+sudo systemctl enable mysql.service
+sudo systemctl start mysql.service
+
+echo "CREATE DATABASE mumble_test_db; "\
+	"CREATE USER 'mumble_test_user'@'localhost' IDENTIFIED BY 'MumbleTestPassword'; "\
+	"GRANT ALL PRIVILEGES ON mumble_test_db.* TO 'mumble_test_user'@'localhost';"  | sudo mysql --user=root --password="root"
+
+
+echo "Configuring PostgreSQL..."
+
+sudo systemctl enable postgresql.service
+sudo systemctl start postgresql.service
+
+echo "CREATE DATABASE mumble_test_db; "\
+	"CREATE USER mumble_test_user ENCRYPTED PASSWORD 'MumbleTestPassword'; "\
+	"GRANT ALL PRIVILEGES ON DATABASE mumble_test_db TO mumble_test_user;" | sudo -u postgres psql

.ci/azure-pipelines/install-environment_macos.bash

@@ -5,6 +5,7 @@
 # that can be found in the LICENSE file at the root of the
 # Mumble source tree or at <https://www.mumble.info/LICENSE>.
 
+
 currentDir=$(pwd)
 cd $AGENT_TEMPDIRECTORY
 
@@ -48,4 +49,19 @@ mkdir -p $MUMBLE_ENVIRONMENT_STORE
 
 chmod +x "$MUMBLE_ENVIRONMENT_PATH/installed/$MUMBLE_ENVIRONMENT_TRIPLET/tools/Ice/slice2cpp"
 
-ls -l $MUMBLE_ENVIRONMENT_STORE
+
+# Setup PostgreSQL database for the Mumble tests
+# Note: we don't configure MySQL as that's not installed on the Azure runners for macOS
+# by default and installing it via homebrew takes forever.
+
+echo "Configuring PostgreSQL..."
+
+# Using brew service start postgresql for some reason does not work so we have
+# to resort to starting PostgreSQL manually.
+postgres_data_dir="$( brew info postgresql | grep -E '^ *initdb' | awk '{print $NF}' )"
+pg_ctl start -D "$postgres_data_dir"
+
+echo "CREATE DATABASE mumble_test_db; "\
+	"CREATE USER mumble_test_user ENCRYPTED PASSWORD 'MumbleTestPassword'; "\
+	"GRANT ALL PRIVILEGES ON DATABASE mumble_test_db TO mumble_test_user;" | psql -d postgres
+

.ci/azure-pipelines/main-pr.yml

@@ -5,7 +5,7 @@
 
 variables:
   MUMBLE_ENVIRONMENT_STORE: '$(Agent.ToolsDirectory)/MumbleBuild'
-  MUMBLE_ENVIRONMENT_SOURCE: 'https://dl.mumble.info/build/vcpkg'
+  MUMBLE_ENVIRONMENT_SOURCE: 'https://github.com/mumble-voip/vcpkg/releases/download/2024-05/'
   MUMBLE_ENVIRONMENT_PATH: '$(MUMBLE_ENVIRONMENT_STORE)/$(MUMBLE_ENVIRONMENT_VERSION)'
   MUMBLE_ENVIRONMENT_TOOLCHAIN: '$(MUMBLE_ENVIRONMENT_PATH)/scripts/buildsystems/vcpkg.cmake'
   MUMBLE_SOURCE_COMMIT: '$(Build.SourceVersion)'
@@ -24,7 +24,7 @@ jobs:
     pool:
       vmImage: 'windows-2022'
     variables:
-      MUMBLE_ENVIRONMENT_VERSION: 'mumble_env.x64-windows-static-md-release.2023-12-31.6a3ce9c65'
+      MUMBLE_ENVIRONMENT_VERSION: 'mumble_env.x64-windows-static-md-release.2024-05-19.5eaf0d45f'
       MUMBLE_ENVIRONMENT_TRIPLET: 'x64-windows-static-md-release'
     steps:
     - template: steps_windows.yml
@@ -45,7 +45,7 @@ jobs:
     pool:
       vmImage: 'macOS-12'
     variables:
-      MUMBLE_ENVIRONMENT_VERSION: 'mumble_env.x64-osx-release.2023-12-31.6a3ce9c65'
+      MUMBLE_ENVIRONMENT_VERSION: 'mumble_env.x64-osx-release.2024-05-19.5eaf0d45f'
       MUMBLE_ENVIRONMENT_TRIPLET: 'x64-osx-release'
     steps:
     - template: steps_macos.yml

.ci/azure-pipelines/main.yml

@@ -5,7 +5,7 @@
 
 variables:
   MUMBLE_ENVIRONMENT_STORE: '$(Agent.ToolsDirectory)/MumbleBuild'
-  MUMBLE_ENVIRONMENT_SOURCE: 'https://dl.mumble.info/build/vcpkg'
+  MUMBLE_ENVIRONMENT_SOURCE: 'https://github.com/mumble-voip/vcpkg/releases/download/2024-05/'
   MUMBLE_ENVIRONMENT_PATH: '$(MUMBLE_ENVIRONMENT_STORE)/$(MUMBLE_ENVIRONMENT_VERSION)'
   MUMBLE_ENVIRONMENT_TOOLCHAIN: '$(MUMBLE_ENVIRONMENT_PATH)/scripts/buildsystems/vcpkg.cmake'
   MUMBLE_SOURCE_COMMIT: '$(Build.SourceVersion)'
@@ -22,7 +22,7 @@ jobs:
     pool:
       vmImage: 'windows-2022'
     variables:
-      MUMBLE_ENVIRONMENT_VERSION: 'mumble_env.x64-windows-static-md-release.2023-12-31.6a3ce9c65'
+      MUMBLE_ENVIRONMENT_VERSION: 'mumble_env.x64-windows-static-md-release.2024-05-19.5eaf0d45f'
       MUMBLE_ENVIRONMENT_TRIPLET: 'x64-windows-static-md-release'
     steps:
     - template: steps_windows.yml
@@ -43,7 +43,7 @@ jobs:
     pool:
       vmImage: 'macOS-12'
     variables:
-      MUMBLE_ENVIRONMENT_VERSION: 'mumble_env.x64-osx-release.2023-12-31.6a3ce9c65'
+      MUMBLE_ENVIRONMENT_VERSION: 'mumble_env.x64-osx-release.2024-05-19.5eaf0d45f'
       MUMBLE_ENVIRONMENT_TRIPLET: 'x64-osx-release'
     steps:
     - template: steps_macos.yml

.ci/azure-pipelines/steps_linux.yml

@@ -12,7 +12,7 @@ steps:
     env:
       MUMBLE_BUILD_NUMBER_TOKEN: $(BUILD_NUMBER_TOKEN)
     displayName: 'Build'
-  - script: 'cd $BUILD_BINARIESDIRECTORY; QT_QPA_PLATFORM=offscreen ctest --output-on-failure'
+  - script: 'cd $BUILD_BINARIESDIRECTORY; QT_QPA_PLATFORM=offscreen ctest --verbose'
     displayName: 'Test'
 # - script: .ci/azure-pipelines/package_AppImage.bash
 #   displayName: 'Release'

.ci/build_windows.bat

@@ -54,7 +54,8 @@ cmake -G Ninja -DCMAKE_TOOLCHAIN_FILE="%MUMBLE_ENVIRONMENT_TOOLCHAIN%" -DVCPKG_T
       -DCMAKE_C_COMPILER=cl.exe -DCMAKE_CXX_COMPILER=cl.exe ^
       -DCMAKE_BUILD_TYPE=Release -DCMAKE_UNITY_BUILD=ON -DBUILD_NUMBER=%BUILD_NUMBER% ^
       -Dpackaging=ON -Dtests=ON -Dstatic=ON -Dsymbols=ON -Dasio=ON -Dg15=ON ^
-      -Ddisplay-install-paths=ON -Delevation=%MUMBLE_USE_ELEVATION% %MUMBLE_SOURCE_REPOSITORY%
+      -Ddisplay-install-paths=ON -Ddatabase-sqlite-tests=ON -Ddatabase-mysql-tests=ON -Ddatabase-postgresql-tests=ON ^
+	  -Delevation=%MUMBLE_USE_ELEVATION% %MUMBLE_SOURCE_REPOSITORY%
 
 if errorlevel 1 (
 	exit /b %errorlevel%

.ci/install-environment_windows.ps1

@@ -22,6 +22,7 @@
 # Always quit on encountered errors
 $ErrorActionPreference = 'Stop'
 
+
 choco install aria2
 
 # We require a separate function that makes sure the run command's exit code
@@ -128,3 +129,31 @@ Invoke-Command 7z x "WixSharp.7z" "-oC:/WixSharp"
 choco install vswhere
 
 Write-Host "Build environment successfully installed"
+
+
+# Setup MySQL and PostgreSQL databases for the Mumble tests
+Write-Host "Configuring PostgreSQL..."
+
+$env:PATH += ";$env:PGBIN"
+$env:PGPASSWORD="root"
+
+pg_ctl start -D "$env:PGDATA"
+
+Write-Output ("CREATE DATABASE mumble_test_db; " +
+	"CREATE USER mumble_test_user ENCRYPTED PASSWORD 'MumbleTestPassword'; " +
+	"GRANT ALL PRIVILEGES ON DATABASE mumble_test_db TO mumble_test_user;") | psql --username "postgres"
+
+
+Write-Host "Configuring MySQL..."
+
+Write-Output "[mysqld]`nlog-bin-trust-function-creators = 1" | Add-Content -Path C:/Windows/my.ini
+
+mysqld --initialize-insecure --console
+Start-Process mysqld
+
+# Give the MySQL daemon some time to start up
+Start-Sleep -Seconds 5
+
+Write-Output ("CREATE DATABASE mumble_test_db; " +
+	"CREATE USER 'mumble_test_user'@'localhost' IDENTIFIED BY 'MumbleTestPassword'; " +
+	"GRANT ALL PRIVILEGES ON mumble_test_db.* TO 'mumble_test_user'@'localhost';")  | mysql --user=root

.github/actions/install-dependencies/install_ubuntu_shared_64bit.sh

@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
 
+set -e
+set -x
+
 sudo apt update
 
 sudo apt -y install \
@@ -29,4 +32,31 @@ sudo apt -y install \
 	libzeroc-ice-dev \
 	zsync \
 	appstream \
-	libpoco-dev
+	libpoco-dev \
+	libsqlite3-dev
+
+# MySQL and PostgreSQL are pre-installed on GitHub-hosted runners. More info about the default setup can be found at
+# - https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-Readme.md#databases
+# - https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#databases
+
+# Setup MySQL and PostgreSQL databases for the Mumble tests
+echo "Configuring MySQL..."
+
+echo -e "[mysqld]\nlog-bin-trust-function-creators = 1" | sudo tee -a /etc/mysql/my.cnf
+
+sudo systemctl enable mysql.service
+sudo systemctl start mysql.service
+
+echo "CREATE DATABASE mumble_test_db; "\
+	"CREATE USER 'mumble_test_user'@'localhost' IDENTIFIED BY 'MumbleTestPassword'; "\
+	"GRANT ALL PRIVILEGES ON mumble_test_db.* TO 'mumble_test_user'@'localhost';"  | sudo mysql --user=root --password="root"
+
+
+echo "Configuring PostgreSQL..."
+
+sudo systemctl enable postgresql.service
+sudo systemctl start postgresql.service
+
+echo "CREATE DATABASE mumble_test_db; "\
+	"CREATE USER mumble_test_user ENCRYPTED PASSWORD 'MumbleTestPassword'; "\
+	"GRANT ALL PRIVILEGES ON DATABASE mumble_test_db TO mumble_test_user;" | sudo -u postgres psql

.github/workflows/build.sh

@@ -17,6 +17,9 @@ cmake -G Ninja \
 	  $CMAKE_OPTIONS \
       -DCMAKE_UNITY_BUILD=ON \
 	  -Ddisplay-install-paths=ON \
+	  -Ddatabase-sqlite-tests=ON \
+	  -Ddatabase-mysql-tests=ON \
+	  -Ddatabase-postgresql-tests=ON \
 	  $ADDITIONAL_CMAKE_OPTIONS \
 	  $VCPKG_CMAKE_OPTIONS
 

.github/workflows/build.yml

@@ -5,8 +5,7 @@ on: [push, pull_request]
 env:
   # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
   BUILD_TYPE: Release
-  CMAKE_OPTIONS: |
-      -Dtests=ON -Dsymbols=ON -Ddisplay-install-paths=ON
+  CMAKE_OPTIONS: -Dtests=ON -Dsymbols=ON -Ddisplay-install-paths=ON -Ddatabase-mysql-tests=ON -Ddatabase-postgresql-tests=ON
   MUMBLE_ENVIRONMENT_SOURCE: 'https://dl.mumble.info/build/vcpkg/'
 
 

.gitmodules

@@ -34,3 +34,6 @@
 [submodule "3rdparty/utfcpp"]
 	path = 3rdparty/utfcpp
 	url = https://github.com/mumble-voip/utfcpp.git
+[submodule "3rdparty/soci"]
+	path = 3rdparty/soci
+	url = https://github.com/Krzmbrzl/soci.git

auxiliary_files/mumble-server.ini

@@ -11,10 +11,22 @@
 ;        NOT variable = """ BUT variable = "\""
 ;        NOT regex = \w* BUT regex = \\w*
 
-; Path to database. If blank, will search for
-; mumble-server.sqlite in default locations or create it if not found.
+; When using SQLite: Path to database. If blank, will search for
+;   mumble-server.sqlite in default locations or create it if not found.
+; Otherwise: The name of the database to use for the Mumble server.
 database=
 
+; Selects the database to use. Possible values are
+; - sqlite (default)
+; - mysql
+; - postgresql
+;dbDriver=postgresql
+
+;dbUsername=
+;dbPassword=
+;dbHost=
+;dbPort=
+
 ; The server defaults to using SQLite with its default rollback journal.
 ; In some situations, using SQLite's write-ahead log (WAL) can be
 ; advantageous.
@@ -25,29 +37,12 @@ database=
 ; values:
 ;
 ; 0 - Use SQLite's default rollback journal.
-; 1 - Use write-ahead log with synchronous=NORMAL.
-;     If the server crashes, the database will be in a consistent state, but
-;     the most recent changes might be lost if the operating system did
-;     not write them to disk yet. This option can improve the server's
-;     interactivity on busy servers, or servers with slow storage.
+; 1 - No longer supported!
 ; 2 - Use write-ahead log with synchronous=FULL.
 ;     All database writes are synchronized to disk when they are made.
 ;     If the server crashes, the database will be include all completed writes.
 ;sqlite_wal=0
 
-; If you wish to use something other than SQLite, you'll need to set the name
-; of the database above, and also uncomment the below.
-; Sticking with SQLite is strongly recommended, as it's the most well tested
-; and by far the fastest solution.
-;
-;dbDriver=QMYSQL
-;dbUsername=
-;dbPassword=
-;dbHost=
-;dbPort=
-;dbPrefix=mumble-server_
-;dbOpts=
-
 ;  The server defaults to not using D-Bus. If you wish to use dbus, which is one of the
 ; RPC methods available in the server, please specify so here.
 ;