[FAL-2031] Add Redis support for Ocim provisioned instances (#822)

Add support for provisioning a redis acl for use by instances as an alternative to rabbitmq.

Co-authored-by: Samuel Walladge <[email protected]>

.env.e2e

@@ -21,6 +21,7 @@ DEFAULT_LOAD_BALANCING_SERVER='[email protected]'
 LOAD_BALANCER_FRAGMENT_NAME_PREFIX='opencraft-'
 DEFAULT_RABBITMQ_API_URL='https://admin:[email protected]:15671'
 DEFAULT_INSTANCE_RABBITMQ_URL='amqps://rabbitmq.example.com:5671'
+DEFAULT_INSTANCE_REDIS_URL='rediss://admin:[email protected]:6397/0'
 CONSUL_ENABLED=false
 CONSUL_SERVERS=haproxy-integration.net.opencraft.hosting
 DISABLE_LOAD_BALANCER_CONFIGURATION=false

.env.test

@@ -19,6 +19,7 @@ DEFAULT_LOAD_BALANCING_SERVER='[email protected]'
 LOAD_BALANCER_FRAGMENT_NAME_PREFIX='opencraft-'
 DEFAULT_RABBITMQ_API_URL='https://admin:[email protected]:15671'
 DEFAULT_INSTANCE_RABBITMQ_URL='amqps://rabbitmq.example.com:5671'
+DEFAULT_INSTANCE_REDIS_URL='rediss://admin:[email protected]:6397/0'
 CONSUL_ENABLED=false
 CONSUL_SERVERS=haproxy-integration.net.opencraft.hosting
 DISABLE_LOAD_BALANCER_CONFIGURATION=false

documentation/configuration.md

@@ -210,6 +210,12 @@ Required settings:
 * `DEFAULT_INSTANCE_RABBITMQ_URL`: The RabbitMQ AMQPS URI to be used by instances. E.g.,
   `amqps://rabbitmq.example.com:5671`
 
+### Redis settings
+* `DEFAULT_INSTANCE_REDIS_URL` The Redis URI (including the protocol, port, db, and basic auth)
+  to be used by instances. E.g., `rediss://admin:[email protected]:6397/0`.
+  Similarly to `DEFAULT_RABBITMQ_API_URL` this setting is saved in the database for new instances
+  using `RedisServerManager._create_default`, where the setting is not explicity overridden.
+
 ### DNS settings
 
 * `DEFAULT_INSTANCE_BASE_DOMAIN`: Instances are created as subdomains of this domain,

documentation/development/docker.md

@@ -6,8 +6,30 @@ This describes how to create an environment for development using Docker.
 
 ## Running OCIM in a docker container
 
-First, set your `.env` file as you would normally,
-but set the following vars:
+First, set your `.env` file as you would normally, but first set the following vars:
+```env
+DEBUG=True
+OPENSTACK_USER='username'
+OPENSTACK_PASSWORD='password'
+OPENSTACK_TENANT='tenant-name'
+OPENSTACK_AUTH_URL='https://auth.cloud.ovh.net/v2.0'
+OPENSTACK_REGION='BHS1'
+OPENSTACK_SANDBOX_SSH_KEYNAME='keypair-name'
+DEFAULT_INSTANCE_BASE_DOMAIN='example.com'
+GANDI_API_KEY='api-key'
+GITHUB_ACCESS_TOKEN='github-token'
+SECRET_KEY='tests'
+DEFAULT_INSTANCE_MYSQL_URL=...
+DEFAULT_RABBITMQ_API_URL=...
+DEFAULT_INSTANCE_RABBITMQ_URL=...
+DEFAULT_INSTANCE_REDIS_URL=...
+DEFAULT_MONGO_REPLICA_SET_USER=...
+DEFAULT_MONGO_REPLICA_SET_PASSWORD=...
+DEFAULT_MONGO_REPLICA_SET_NAME=...
+DEFAULT_MONGO_REPLICA_SET_PRIMARY=...
+DEFAULT_MONGO_REPLICA_SET_HOSTS=...
+REDIS_URL=...
+```
 
 ```sh
 ALLOWED_HOSTS='["*"]'
@@ -43,3 +65,4 @@ For example:
 ```sh
 docker-compose logs -f ocim
 ```
+

instance/admin.py

@@ -33,6 +33,7 @@
 from instance.models.openedx_deployment import OpenEdXDeployment
 from instance.models.openedx_instance import OpenEdXInstance
 from instance.models.rabbitmq_server import RabbitMQServer
+from instance.models.redis_server import RedisServer
 from instance.models.server import OpenStackServer
 
 
@@ -109,6 +110,10 @@ class RabbitMQServerAdmin(admin.ModelAdmin): # pylint: disable=missing-docstring
     list_display = ('name', 'description', 'api_url', 'instance_host', 'instance_port')
 
 
+class RedisServerAdmin(admin.ModelAdmin): # pylint: disable=missing-docstring
+    list_display = ('name', 'description', 'instance_host', 'instance_port', 'instance_db')
+
+
 class LoadBalancingServerAdmin(admin.ModelAdmin): # pylint: disable=missing-docstring
     list_display = ('domain', 'ssh_username')
 
@@ -132,5 +137,6 @@ def has_changes(self, obj):
 admin.site.register(MongoDBServer, MongoDBServerAdmin)
 admin.site.register(MongoDBReplicaSet, MongoDBReplicaSetAdmin)
 admin.site.register(RabbitMQServer, RabbitMQServerAdmin)
+admin.site.register(RedisServer, RedisServerAdmin)
 admin.site.register(LoadBalancingServer, LoadBalancingServerAdmin)
 admin.site.register(OpenEdXDeployment, OpenEdXDeploymentAdmin)

instance/migrations/0142_add_cache_db_to_openedxinstance.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.24 on 2021-08-13 08:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('instance', '0141_features_to_features_extras'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='openedxinstance',
+            name='cache_db',
+            field=models.CharField(choices=[('redis', 'redis'), ('rabbit_mq', 'rabbit_mq')], default='rabbit_mq', max_length=9),
+        ),
+    ]

instance/migrations/0143_add_redis_server.py

@@ -0,0 +1,58 @@
+# Generated by Django 2.2.24 on 2021-08-13 08:53
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+import instance.models.mixins.redis
+import instance.models.utils
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('instance', '0142_add_cache_db_to_openedxinstance'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RedisServer',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
+                ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
+                ('name', models.CharField(max_length=250)),
+                ('description', models.CharField(blank=True, max_length=250)),
+                ('admin_username', models.CharField(max_length=64)),
+                ('admin_password', models.CharField(max_length=128)),
+                ('instance_host', models.CharField(max_length=128)),
+                ('instance_port', models.PositiveIntegerField(default=5671)),
+                ('instance_db', models.PositiveIntegerField(default=0)),
+                ('use_ssl_connections', models.BooleanField(default=True)),
+                ('accepts_new_clients', models.BooleanField(default=False)),
+            ],
+            options={
+                'verbose_name': 'Redis Server',
+            },
+            bases=(instance.models.utils.ValidateModelMixin, models.Model),
+        ),
+        migrations.AddField(
+            model_name='openedxinstance',
+            name='redis_password',
+            field=models.CharField(max_length=64, null=True),
+        ),
+        migrations.AddField(
+            model_name='openedxinstance',
+            name='redis_provisioned',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='openedxinstance',
+            name='redis_username',
+            field=models.CharField(max_length=32, null=True),
+        ),
+        migrations.AddField(
+            model_name='openedxinstance',
+            name='redis_server',
+            field=models.ForeignKey(blank=True, default=instance.models.mixins.redis.select_random_redis_server, null=True, on_delete=django.db.models.deletion.PROTECT, to='instance.RedisServer'),
+        ),
+    ]

instance/migrations/0144_make_redis_username_unique.py

@@ -0,0 +1,37 @@
+# Generated by Django 2.2.24 on 2021-08-13 08:53
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+import instance.models.mixins.redis
+import instance.models.utils
+
+
+def generate_credentials(apps, schema_editor):
+    Model = apps.get_model('instance', 'OpenEdXInstance')
+    for row in Model.objects.all():
+        row.redis_username = instance.models.mixins.redis.random_username()
+        row.redis_password = instance.models.mixins.redis.random_password()
+        row.save(update_fields=['redis_username', 'redis_password'])
+
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('instance', '0143_add_redis_server'),
+    ]
+
+    operations = [
+        migrations.RunPython(generate_credentials, reverse_code=migrations.RunPython.noop),
+        migrations.AlterField(
+            model_name='openedxinstance',
+            name='redis_password',
+            field=models.CharField(default=instance.models.mixins.redis.random_password, max_length=64),
+        ),
+        migrations.AlterField(
+            model_name='openedxinstance',
+            name='redis_username',
+            field=models.CharField(default=instance.models.mixins.redis.random_username, max_length=32, unique=True),
+        ),
+    ]

instance/models/mixins/openedx_database.py

@@ -22,11 +22,13 @@
 import hashlib
 import hmac
 
+from django.db import models
 import MySQLdb as mysql
 import yaml
 
 from instance.models.mixins.database import MySQLInstanceMixin, MongoDBInstanceMixin
 from instance.models.mixins.rabbitmq import RabbitMQInstanceMixin
+from instance.models.mixins.redis import RedisInstanceMixin
 
 FORUM_MONGO_REPLICA_SET_URL = (
     "mongodb://{{ FORUM_MONGO_USER }}:{{ FORUM_MONGO_PASSWORD }}@"
@@ -41,13 +43,25 @@
 # Classes #####################################################################
 
 
-class OpenEdXDatabaseMixin(MySQLInstanceMixin, MongoDBInstanceMixin, RabbitMQInstanceMixin):
+class OpenEdXDatabaseMixin(MySQLInstanceMixin, MongoDBInstanceMixin, RabbitMQInstanceMixin, RedisInstanceMixin):
     """
     Mixin that provides functionality required for the database backends that an
     OpenEdX Instance uses
 
+
     TODO: ElasticSearch?
     """
+
+    REDIS = "redis"
+    RABBIT_MQ = "rabbit_mq"
+    CACHE_DBS = (
+        (REDIS, REDIS),
+        (RABBIT_MQ, RABBIT_MQ),
+    )
+
+    # TODO: When every server is using REDIS, use REDIS as default
+    cache_db = models.CharField(max_length=9, choices=CACHE_DBS, default=RABBIT_MQ)
+
     class Meta:
         abstract = True
 
@@ -475,6 +489,39 @@ def _get_rabbitmq_settings(self):
             "EDXAPP_CELERY_BROKER_USE_SSL": True
         }
 
+    def _get_redis_settings(self):
+        """
+        Return dictionary of Redis settings
+        """
+
+        redis_hostname = "{}:{}".format(
+            self.redis_server.instance_host,
+            self.redis_server.instance_port
+        )
+
+        redis_transport_options = {
+            "CELERY_BROKER_TRANSPORT_OPTIONS": {
+                "global_keyprefix": self.redis_key_prefix
+            }
+        }
+
+        return {
+            "EDXAPP_REDIS_HOSTNAME": redis_hostname,
+            "EDXAPP_CELERY_BROKER_TRANSPORT": "redis",
+            "EDXAPP_CELERY_USER": self.redis_username,
+            "EDXAPP_CELERY_PASSWORD": self.redis_password,
+            "EDXAPP_CELERY_BROKER_HOSTNAME": redis_hostname,
+            "EDXAPP_CELERY_BROKER_VHOST": str(self.redis_server.instance_db),
+            "EDXAPP_CELERY_BROKER_USE_SSL": self.redis_server.use_ssl_connections,
+
+            "EDXAPP_LMS_ENV_EXTRA": {
+                **redis_transport_options
+            },
+            "EDXAPP_CMS_ENV_EXTRA": {
+                **redis_transport_options
+            }
+        }
+
     def get_database_settings(self):
         """
         Get configuration_database_settings to pass to a new AppServer
@@ -489,7 +536,13 @@ def get_database_settings(self):
         if self.mongodb_replica_set or self.mongodb_server:
             new_settings.update(self._get_mongo_settings())
 
-        # RabbitMQ:
-        new_settings.update(self._get_rabbitmq_settings())
+        if self.cache_db == self.REDIS:
+            # Redis:
+            new_settings.update(self._get_redis_settings())
+        elif self.cache_db == self.RABBIT_MQ:
+            # RabbitMQ:
+            new_settings.update(self._get_rabbitmq_settings())
+        else:
+            raise NotImplementedError(f"{self.cache_db} does not load any cache DB settings")
 
         return yaml.dump(new_settings, default_flow_style=False)