diff --git a/kube/helm/templates/crds/shulkermc.io_minecraftserverfleets.yaml b/kube/helm/templates/crds/shulkermc.io_minecraftserverfleets.yaml index 8d1e499f..aefcfa49 100644 --- a/kube/helm/templates/crds/shulkermc.io_minecraftserverfleets.yaml +++ b/kube/helm/templates/crds/shulkermc.io_minecraftserverfleets.yaml @@ -283,6 +283,13 @@ spec: description: Name of an optional ConfigMap already containing the server configuration nullable: true type: string + lifecycleStrategy: + default: AllocateWhenNotEmpty + description: Strategy to apply concerning Agones `GameServer` lifecycle management + enum: + - AllocateWhenNotEmpty + - Manual + type: string maxPlayers: default: 20 description: Number of maximum players that can connect to the MinecraftServer Deployment diff --git a/kube/helm/templates/crds/shulkermc.io_minecraftservers.yaml b/kube/helm/templates/crds/shulkermc.io_minecraftservers.yaml index de4c1b54..609018d0 100644 --- a/kube/helm/templates/crds/shulkermc.io_minecraftservers.yaml +++ b/kube/helm/templates/crds/shulkermc.io_minecraftservers.yaml @@ -53,6 +53,13 @@ spec: description: Name of an optional ConfigMap already containing the server configuration nullable: true type: string + lifecycleStrategy: + default: AllocateWhenNotEmpty + description: Strategy to apply concerning Agones `GameServer` lifecycle management + enum: + - AllocateWhenNotEmpty + - Manual + type: string maxPlayers: default: 20 description: Number of maximum players that can connect to the MinecraftServer Deployment diff --git a/kube/helm/templates/crds/shulkermc.io_proxyfleets.yaml b/kube/helm/templates/crds/shulkermc.io_proxyfleets.yaml index be97ab93..c5d61830 100644 --- a/kube/helm/templates/crds/shulkermc.io_proxyfleets.yaml +++ b/kube/helm/templates/crds/shulkermc.io_proxyfleets.yaml @@ -337,6 +337,12 @@ spec: type: object nullable: true type: array + playersDeltaBeforeExclusion: + default: 15 + description: Number of player slots to reserve when exclusing a proxy from the load balancer. This will allow load balancer implementations to update itself while still being able to accept some players + format: uint32 + minimum: 0.0 + type: integer plugins: description: List of references to plugins to download items: diff --git a/packages/shulker-crds/src/v1alpha1/proxy_fleet.rs b/packages/shulker-crds/src/v1alpha1/proxy_fleet.rs index 4e78f993..83b966f2 100644 --- a/packages/shulker-crds/src/v1alpha1/proxy_fleet.rs +++ b/packages/shulker-crds/src/v1alpha1/proxy_fleet.rs @@ -130,6 +130,15 @@ pub struct ProxyFleetTemplateConfigurationSpec { /// drained automatically #[schemars(default = "ProxyFleetTemplateConfigurationSpec::default_ttl_seconds")] pub ttl_seconds: u32, + + /// Number of player slots to reserve when exclusing a proxy + /// from the load balancer. This will allow load balancer + /// implementations to update itself while still being able + /// to accept some players + #[schemars( + default = "ProxyFleetTemplateConfigurationSpec::default_players_delta_before_exclusion" + )] + pub players_delta_before_exclusion: u32, } #[cfg(not(tarpaulin_include))] @@ -153,6 +162,10 @@ impl ProxyFleetTemplateConfigurationSpec { fn default_ttl_seconds() -> u32 { 86400 } + + fn default_players_delta_before_exclusion() -> u32 { + 15 + } } #[derive(Deserialize, Serialize, Clone, Debug, Default, JsonSchema)] diff --git a/packages/shulker-operator/assets/proxy-probe-readiness.sh b/packages/shulker-operator/assets/proxy-probe-readiness.sh index 0f48adc2..2741ec85 100644 --- a/packages/shulker-operator/assets/proxy-probe-readiness.sh +++ b/packages/shulker-operator/assets/proxy-probe-readiness.sh @@ -4,6 +4,8 @@ set -o xtrace if [ -f "/tmp/drain-lock" ]; then echo "Drain lock found" && exit 1 +elif [ -f "/tmp/readiness-lock" ]; then + echo "Readiness lock found" && exit 1 fi bash /usr/bin/health.sh diff --git a/packages/shulker-operator/src/reconcilers/proxy_fleet/config_map.rs b/packages/shulker-operator/src/reconcilers/proxy_fleet/config_map.rs index 222c92e4..f557eea0 100644 --- a/packages/shulker-operator/src/reconcilers/proxy_fleet/config_map.rs +++ b/packages/shulker-operator/src/reconcilers/proxy_fleet/config_map.rs @@ -348,6 +348,7 @@ mod bungeecord { server_icon: "A Server Icon".to_string(), proxy_protocol: true, ttl_seconds: 300, + players_delta_before_exclusion: 15, }; let service_spec = Some(ProxyFleetServiceSpec { type_: ProxyFleetServiceType::LoadBalancer, @@ -552,6 +553,7 @@ mod velocity { server_icon: "A Server Icon".to_string(), proxy_protocol: true, ttl_seconds: 300, + players_delta_before_exclusion: 15, }; let service_spec = Some(ProxyFleetServiceSpec { type_: ProxyFleetServiceType::LoadBalancer, diff --git a/packages/shulker-operator/src/reconcilers/proxy_fleet/fixtures.rs b/packages/shulker-operator/src/reconcilers/proxy_fleet/fixtures.rs index 2220cf08..805c1d82 100644 --- a/packages/shulker-operator/src/reconcilers/proxy_fleet/fixtures.rs +++ b/packages/shulker-operator/src/reconcilers/proxy_fleet/fixtures.rs @@ -64,6 +64,7 @@ lazy_static! { server_icon: "abc==".to_string(), proxy_protocol: true, ttl_seconds: 3600, + players_delta_before_exclusion: 15, }, pod_overrides: Some(ProxyFleetTemplatePodOverridesSpec { image: None, diff --git a/packages/shulker-operator/src/reconcilers/proxy_fleet/fleet.rs b/packages/shulker-operator/src/reconcilers/proxy_fleet/fleet.rs index 086ae18b..0fa5f492 100644 --- a/packages/shulker-operator/src/reconcilers/proxy_fleet/fleet.rs +++ b/packages/shulker-operator/src/reconcilers/proxy_fleet/fleet.rs @@ -447,6 +447,11 @@ impl<'a> FleetBuilder { value: Some(spec.config.ttl_seconds.to_string()), ..EnvVar::default() }, + EnvVar { + name: "SHULKER_PROXY_PLAYER_DELTA_BEFORE_EXCLUSION".to_string(), + value: Some(spec.config.players_delta_before_exclusion.to_string()), + ..EnvVar::default() + }, EnvVar { name: "SHULKER_NETWORK_ADMINS".to_string(), value: Some( diff --git a/packages/shulker-operator/src/reconcilers/proxy_fleet/snapshots/shulker_operator__reconcilers__proxy_fleet__config_map__tests__build_snapshot.snap b/packages/shulker-operator/src/reconcilers/proxy_fleet/snapshots/shulker_operator__reconcilers__proxy_fleet__config_map__tests__build_snapshot.snap index 6c325f3f..78ed6c33 100644 --- a/packages/shulker-operator/src/reconcilers/proxy_fleet/snapshots/shulker_operator__reconcilers__proxy_fleet__config_map__tests__build_snapshot.snap +++ b/packages/shulker-operator/src/reconcilers/proxy_fleet/snapshots/shulker_operator__reconcilers__proxy_fleet__config_map__tests__build_snapshot.snap @@ -6,7 +6,7 @@ apiVersion: v1 kind: ConfigMap data: init-fs.sh: "#!/bin/sh\nset -euo pipefail\nset -o xtrace\n\ncp \"${SHULKER_CONFIG_DIR}/probe-readiness.sh\" \"${SHULKER_PROXY_DATA_DIR}/probe-readiness.sh\"\ncat \"${SHULKER_CONFIG_DIR}/server-icon.png\" | base64 -d > \"${SHULKER_PROXY_DATA_DIR}/server-icon.png\"\n\nif [ \"${SHULKER_VERSION_CHANNEL}\" == \"Velocity\" ]; then\n cp \"${SHULKER_CONFIG_DIR}/velocity-config.toml\" \"${SHULKER_PROXY_DATA_DIR}/velocity.toml\"\n echo \"dummy\" > \"${SHULKER_PROXY_DATA_DIR}/forwarding.secret\"\nelse\n cp \"${SHULKER_CONFIG_DIR}/bungeecord-config.yml\" \"${SHULKER_PROXY_DATA_DIR}/config.yml\"\nfi\n\nif [ ! -z \"${SHULKER_PROXY_PLUGIN_URLS+x}\" ]; then\n mkdir -p \"${SHULKER_PROXY_DATA_DIR}/plugins\"\n for plugin_url in ${SHULKER_PROXY_PLUGIN_URLS//;/ }; do\n (cd \"${SHULKER_PROXY_DATA_DIR}/plugins\" && wget \"${plugin_url}\")\n done\nfi\n\nif [ ! -z \"${SHULKER_PROXY_PATCH_URLS+x}\" ]; then\n for patch_url in ${SHULKER_PROXY_PATCH_URLS//;/ }; do\n (cd \"${SHULKER_PROXY_DATA_DIR}\" && wget \"${patch_url}\" -O - | tar -xzv)\n done\nfi\n" - probe-readiness.sh: "#!/bin/sh\nset -euo pipefail\nset -o xtrace\n\nif [ -f \"/tmp/drain-lock\" ]; then\n echo \"Drain lock found\" && exit 1\nfi\n\nbash /usr/bin/health.sh\n" + probe-readiness.sh: "#!/bin/sh\nset -euo pipefail\nset -o xtrace\n\nif [ -f \"/tmp/drain-lock\" ]; then\n echo \"Drain lock found\" && exit 1\nelif [ -f \"/tmp/readiness-lock\" ]; then\n echo \"Readiness lock found\" && exit 1\nfi\n\nbash /usr/bin/health.sh\n" server-icon.png: abc== velocity-config.toml: "config-version = \"2.6\"\nbind = \"0.0.0.0:25577\"\nmotd = \"A Motd\"\nshow-max-players = 1000\nonline-mode = true\nforce-key-authentication = true\nprevent-client-proxy-connections = true\nforwarding-secret-file = \"/mnt/shulker/forwarding-secret/key\"\nplayer-info-forwarding-mode = \"modern\"\n\n[servers]\nlobby = \"localhost:30000\"\nlimbo = \"localhost:30001\"\ntry = [\"lobby\", \"limbo\"]\n\n[forced-hosts]\n\n[advanced]\nhaproxy-protocol = true\ntcp-fast-open = true\n\n" metadata: diff --git a/packages/shulker-operator/src/reconcilers/proxy_fleet/snapshots/shulker_operator__reconcilers__proxy_fleet__fleet__tests__build_snapshot.snap b/packages/shulker-operator/src/reconcilers/proxy_fleet/snapshots/shulker_operator__reconcilers__proxy_fleet__fleet__tests__build_snapshot.snap index 6bdaedeb..999ad23c 100644 --- a/packages/shulker-operator/src/reconcilers/proxy_fleet/snapshots/shulker_operator__reconcilers__proxy_fleet__fleet__tests__build_snapshot.snap +++ b/packages/shulker-operator/src/reconcilers/proxy_fleet/snapshots/shulker_operator__reconcilers__proxy_fleet__fleet__tests__build_snapshot.snap @@ -73,6 +73,8 @@ spec: fieldPath: metadata.namespace - name: SHULKER_PROXY_TTL_SECONDS value: "3600" + - name: SHULKER_PROXY_PLAYER_DELTA_BEFORE_EXCLUSION + value: "15" - name: SHULKER_NETWORK_ADMINS value: "" - name: SHULKER_PROXY_REDIS_HOST diff --git a/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/Configuration.kt b/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/Configuration.kt index 0fa7872d..cf083e10 100644 --- a/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/Configuration.kt +++ b/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/Configuration.kt @@ -2,13 +2,17 @@ package io.shulkermc.proxyagent import java.util.Optional import java.util.UUID +import kotlin.jvm.optionals.getOrDefault +@SuppressWarnings("detekt:MagicNumber") object Configuration { val CLUSTER_NAME = getStringEnv("SHULKER_CLUSTER_NAME") val PROXY_NAMESPACE = getStringEnv("SHULKER_PROXY_NAMESPACE") val PROXY_NAME = getStringEnv("SHULKER_PROXY_NAME") val PROXY_TTL_SECONDS = getLongEnv("SHULKER_PROXY_TTL_SECONDS") + val PROXY_PLAYER_DELTA_BEFORE_EXCLUSION = getOptionalIntEnv("SHULKER_PROXY_PLAYER_DELTA_BEFORE_EXCLUSION") + .getOrDefault(15) val NETWORK_ADMINS: List = getOptionalStringEnv("SHULKER_NETWORK_ADMINS") .map { @@ -26,5 +30,7 @@ object Configuration { private fun getStringEnv(name: String): String = requireNotNull(System.getenv(name)) { "Missing $name" } private fun getOptionalStringEnv(name: String): Optional = Optional.ofNullable(System.getenv(name)) private fun getIntEnv(name: String): Int = getStringEnv(name).toInt() + private fun getOptionalIntEnv(name: String): Optional = Optional.ofNullable(System.getenv(name)) + .map { it.toInt() } private fun getLongEnv(name: String): Long = getStringEnv(name).toLong() } diff --git a/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/ShulkerProxyAgentCommon.kt b/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/ShulkerProxyAgentCommon.kt index f7c0ca2e..c181a7d9 100644 --- a/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/ShulkerProxyAgentCommon.kt +++ b/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/ShulkerProxyAgentCommon.kt @@ -87,7 +87,7 @@ class ShulkerProxyAgentCommon(val proxyInterface: ProxyInterface, val logger: Lo } this.cache.registerProxy(Configuration.PROXY_NAME, this.proxyInterface.getPlayerCapacity()) - this.agonesGateway.setAllocated() + this.agonesGateway.setReady() } catch (@Suppress("TooGenericExceptionCaught") e: Exception) { this.logger.log(Level.SEVERE, "Shulker Agent crashed, stopping proxy", e) this.shutdown() diff --git a/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/adapters/filesystem/FileSystemAdapter.kt b/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/adapters/filesystem/FileSystemAdapter.kt index d1994fba..debdf990 100644 --- a/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/adapters/filesystem/FileSystemAdapter.kt +++ b/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/adapters/filesystem/FileSystemAdapter.kt @@ -1,5 +1,8 @@ package io.shulkermc.proxyagent.adapters.filesystem interface FileSystemAdapter { - fun createDrainFile() + fun createDrainLock() + + fun createReadinessLock() + fun deleteReadinessLock() } diff --git a/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/adapters/filesystem/LocalFileSystemAdapter.kt b/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/adapters/filesystem/LocalFileSystemAdapter.kt index 52331296..9190ca9a 100644 --- a/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/adapters/filesystem/LocalFileSystemAdapter.kt +++ b/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/adapters/filesystem/LocalFileSystemAdapter.kt @@ -4,11 +4,22 @@ import java.nio.file.Files import java.nio.file.Path private val DRAIN_LOCK_PATH = Path.of("/tmp/drain-lock") +private val READINESS_LOCK_PATH = Path.of("/tmp/readiness-lock") class LocalFileSystemAdapter : FileSystemAdapter { - override fun createDrainFile() { + override fun createDrainLock() { if (!Files.exists(DRAIN_LOCK_PATH)) { Files.createFile(DRAIN_LOCK_PATH) } } + + override fun createReadinessLock() { + if (!Files.exists(READINESS_LOCK_PATH)) { + Files.createFile(READINESS_LOCK_PATH) + } + } + + override fun deleteReadinessLock() { + Files.deleteIfExists(READINESS_LOCK_PATH) + } } diff --git a/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/services/PlayerMovementService.kt b/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/services/PlayerMovementService.kt index d420a95d..c73aa691 100644 --- a/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/services/PlayerMovementService.kt +++ b/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/services/PlayerMovementService.kt @@ -31,6 +31,9 @@ class PlayerMovementService(private val agent: ShulkerProxyAgentCommon) { ) } + private val maxPlayersWithExclusionDelta = + this.agent.proxyInterface.getPlayerCapacity() - Configuration.PROXY_PLAYER_DELTA_BEFORE_EXCLUSION + private val onlinePlayerCountSupplier = Suppliers.memoizeWithExpiration( { this.agent.cache.countOnlinePlayers() }, ONLINE_PLAYERS_COUNT_MEMOIZE_SECONDS, @@ -42,6 +45,7 @@ class PlayerMovementService(private val agent: ShulkerProxyAgentCommon) { java.util.concurrent.TimeUnit.SECONDS ) + private var isAllocatedByAgones = false private var acceptingPlayers = true init { @@ -57,8 +61,10 @@ class PlayerMovementService(private val agent: ShulkerProxyAgentCommon) { this.acceptingPlayers = acceptingPlayers if (acceptingPlayers) { + this.agent.fileSystem.deleteReadinessLock() this.agent.logger.info("Proxy is now accepting players") } else { + this.agent.fileSystem.createReadinessLock() this.agent.logger.info("Proxy is no longer accepting players") } } @@ -71,15 +77,34 @@ class PlayerMovementService(private val agent: ShulkerProxyAgentCommon) { if (!this.acceptingPlayers) { return PlayerPreLoginHookResult.disallow(MSG_NOT_ACCEPTING_PLAYERS) } + return PlayerPreLoginHookResult.allow() } private fun onPlayerLogin(player: Player) { this.agent.cache.updateCachedPlayerName(player.uniqueId, player.name) + + if (!this.isAllocatedByAgones) { + this.isAllocatedByAgones = true + this.agent.agonesGateway.setAllocated() + } + + if (this.isProxyConsideredFull()) { + this.setAcceptingPlayers(false) + } } private fun onPlayerDisconnect(player: Player) { this.agent.cache.unsetPlayerPosition(player.uniqueId) + + if (this.isAllocatedByAgones && this.agent.proxyInterface.getPlayerCount() == 0) { + this.isAllocatedByAgones = false + this.agent.agonesGateway.setReady() + } + + if (!this.isProxyConsideredFull()) { + this.setAcceptingPlayers(true) + } } private fun onServerPreConnect(player: Player, originalServerName: String): ServerPreConnectHookResult { @@ -105,4 +130,8 @@ class PlayerMovementService(private val agent: ShulkerProxyAgentCommon) { private fun onServerPostConnect(player: Player, serverName: String) { this.agent.cache.setPlayerPosition(player.uniqueId, Configuration.PROXY_NAME, serverName) } + + private fun isProxyConsideredFull(): Boolean { + return this.agent.proxyInterface.getPlayerCount() >= this.maxPlayersWithExclusionDelta + } } diff --git a/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/services/ProxyLifecycleService.kt b/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/services/ProxyLifecycleService.kt index 540d763d..4462cf6a 100644 --- a/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/services/ProxyLifecycleService.kt +++ b/packages/shulker-proxy-agent/src/common/kotlin/io/shulkermc/proxyagent/services/ProxyLifecycleService.kt @@ -43,7 +43,7 @@ class ProxyLifecycleService(private val agent: ShulkerProxyAgentCommon) { } this.drained = true - this.agent.fileSystem.createDrainFile() + this.agent.fileSystem.createDrainLock() this.agent.playerMovementService.setAcceptingPlayers(false) this.agent.proxyInterface.scheduleRepeatingTask(