Merge pull request #6888 from appwrite/feat-db-per-process

Feat db per worker
appwrite · Oct 12, 2023 · 4e988dd · 4e988dd
2 parents 2cd4514 + 48bc369
commit 4e988dd
Show file tree

Hide file tree

Showing 7 changed files with 35 additions and 21 deletions.
diff --git a/app/views/install/compose.phtml b/app/views/install/compose.phtml
@@ -332,6 +332,8 @@ services:
       - mariadb
     environment:
       - _APP_ENV
+      - _APP_WORKERS_NUM=1
+      - _APP_QUEUE_NAME=database_db_main
       - _APP_WORKER_PER_CORE
       - _APP_OPENSSL_KEY_V1
       - _APP_REDIS_HOST

diff --git a/app/worker.php b/app/worker.php
@@ -18,13 +18,11 @@
 use Utopia\App;
 use Utopia\Cache\Adapter\Sharding;
 use Utopia\Cache\Cache;
-use Utopia\CLI\CLI;
 use Utopia\CLI\Console;
 use Utopia\Config\Config;
 use Utopia\Database\Database;
 use Utopia\Database\Document;
 use Utopia\Database\Validator\Authorization;
-use Utopia\Queue\Adapter\Swoole;
 use Utopia\Platform\Service;
 use Utopia\Queue\Message;
 use Utopia\Queue\Server;
@@ -134,9 +132,6 @@
 Server::setResource('queueForBuilds', function (Connection $queue) {
     return new Build($queue);
 }, ['queue']);
-Server::setResource('queueForDatabase', function (Connection $queue) {
-    return new EventDatabase($queue);
-}, ['queue']);
 Server::setResource('queueForDeletes', function (Connection $queue) {
     return new Delete($queue);
 }, ['queue']);
@@ -216,17 +211,31 @@
 $platform = new Appwrite();
 $args = $_SERVER['argv'];
 
-if (isset($args[0])) {
-    $workerName = end($args);
-} else {
+if (!isset($args[1])) {
     Console::error('Missing worker name');
+    Console::exit(1);
+}
+
+\array_shift($args);
+$workerName = $args[0];
+$workerIndex = $args[1] ?? '';
+
+if (!empty($workerNum)) {
+    $workerName .= '_' . $workerIndex;
 }
 
 try {
+    /**
+     * Any worker can be configured with the following env vars:
+     * - _APP_WORKERS_NUM           The total number of worker processes
+     * - _APP_WORKER_PER_CORE       The number of worker processes per core (ignored if _APP_WORKERS_NUM is set)
+     * - _APP_QUEUE_NAME  The name of the queue to read for database events
+     */
     $platform->init(Service::TYPE_WORKER, [
-        'workersNum' => strtolower($workerName) === 'databases'? 1 :swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6)),
+        'workersNum' => App::getEnv('_APP_WORKERS_NUM', swoole_cpu_num() * intval(App::getEnv('_APP_WORKER_PER_CORE', 6))),
         'connection' => $pools->get('queue')->pop()->getResource(),
         'workerName' => strtolower($workerName) ?? null,
+        'queueName' => App::getEnv('_APP_QUEUE_NAME', 'v1-' . strtolower($workerName))
     ]);
 } catch (\Exception $e) {
     Console::error($e->getMessage() . ', File: ' . $e->getFile() .  ', Line: ' . $e->getLine());

diff --git a/bin/worker-databases b/bin/worker-databases
@@ -1,3 +1,3 @@
 #!/bin/sh
 
-php /usr/src/code/app/worker.php databases $@
+php /usr/src/code/app/worker.php databases $@
diff --git a/composer.json b/composer.json
@@ -56,15 +56,15 @@
         "utopia-php/image": "0.5.*",
         "utopia-php/locale": "0.4.*",
         "utopia-php/logger": "0.3.*",
-        "utopia-php/messaging": "0.1.*",
+        "utopia-php/messaging": "0.2.*",
         "utopia-php/migration": "0.3.*",
         "utopia-php/orchestration": "0.9.*",
         "utopia-php/platform": "dev-integrate-workers as 0.3.3",
         "utopia-php/pools": "0.4.*",
         "utopia-php/preloader": "0.2.*",
         "utopia-php/queue": "dev-feat-get-worker-start as 0.5.3",
         "utopia-php/registry": "0.5.*",
-        "utopia-php/storage": "0.14.*",
+        "utopia-php/storage": "0.17.*",
         "utopia-php/swoole": "0.5.*",
         "utopia-php/vcs": "0.5.*",
         "utopia-php/websocket": "0.1.*",

diff --git a/composer.lock b/composer.lock
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -372,6 +372,8 @@ services:
       - mariadb
     environment:
       - _APP_ENV
+      - _APP_WORKERS_NUM=1
+      - _APP_QUEUE_NAME=database_db_main
       - _APP_WORKER_PER_CORE
       - _APP_OPENSSL_KEY_V1
       - _APP_REDIS_HOST

diff --git a/src/Appwrite/Event/Database.php b/src/Appwrite/Event/Database.php
@@ -2,6 +2,7 @@
 
 namespace Appwrite\Event;
 
+use Utopia\App;
 use Utopia\Database\Document;
 use Utopia\Queue\Client;
 use Utopia\Queue\Connection;
@@ -17,9 +18,7 @@ public function __construct(protected Connection $connection)
     {
         parent::__construct($connection);
 
-        $this
-            ->setQueue(Event::DATABASE_QUEUE_NAME)
-            ->setClass(Event::DATABASE_CLASS_NAME);
+        $this->setClass(Event::DATABASE_CLASS_NAME);
     }
 
     /**
@@ -109,6 +108,8 @@ public function getDocument(): ?Document
      */
     public function trigger(): string|bool
     {
+        $this->setQueue($this->getProject()->getAttribute('database'));
+
         $client = new Client($this->queue, $this->connection);
 
         return $client->enqueue([