redis-stack in clinterwebz

docker-compose.yml

@@ -8,6 +8,7 @@ services:
       - 5000:5000
     depends_on:
       - redis-unstable
+      - redis-stack
 
   flask-nginx:
     build:
@@ -23,4 +24,11 @@ services:
       dockerfile: docker/redis/Dockerfile
       context: .
     ports:
-      - 6379:6379
+      - 6379:6379
+
+  redis-stack:
+    build:
+      dockerfile: docker/redis-stack/Dockerfile
+      context: .
+    ports:
+      - 6378:6379

docker/flask/settings.json

@@ -8,6 +8,10 @@
         {
             "id": "default",
             "url": "redis://interwebz:password1@redis-unstable:6379"
+        },
+        {
+            "id": "stack",
+            "url": "redis://interwebz:password1@redis-stack:6379"
         }
     ],
     "SECRET_KEY": "some_random_value",

docker/redis-stack/Dockerfile

@@ -0,0 +1,37 @@
+FROM redis/redis-stack-server
+RUN groupadd -r -g 999 redis && useradd -r -g redis -u 999 redis
+
+ENV GOSU_VERSION 1.12
+RUN set -eux; \
+	savedAptMark="$(apt-mark showmanual)"; \
+	apt-get update; \
+	apt-get install -y --no-install-recommends ca-certificates dirmngr gnupg wget; \
+	rm -rf /var/lib/apt/lists/*; \
+	dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \
+	wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \
+	wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \
+	export GNUPGHOME="$(mktemp -d)"; \
+	gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \
+	gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \
+	gpgconf --kill all; \
+	rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \
+	apt-mark auto '.*' > /dev/null; \
+	[ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \
+	apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \
+	chmod +x /usr/local/bin/gosu; \
+	gosu --version; \
+	gosu nobody true
+
+RUN mkdir /etc/redis
+COPY /docker/redis-stack/redis.conf /etc/redis
+COPY /docker/redis-stack/interwebz.aclfile /etc/redis
+
+RUN chown redis:redis /data
+VOLUME /data
+WORKDIR /data
+
+COPY /docker/redis/docker-entrypoint.sh /usr/local/bin/
+ENTRYPOINT ["docker-entrypoint.sh"]
+
+EXPOSE 6379
+CMD ["redis-server","/etc/redis/redis.conf"]

docker/redis-stack/docker-entrypoint.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+set -e
+
+# first arg is `-f` or `--some-option`
+# or first arg is `something.conf`
+if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then
+	set -- redis-server "$@"
+fi
+
+# allow the container to be started with `--user`
+if [ "$1" = 'redis-server' -a "$(id -u)" = '0' ]; then
+	find . \! -user redis -exec chown redis '{}' +
+	exec gosu redis "$0" "$@"
+fi
+
+exec "$@"

docker/redis-stack/interwebz.aclfile

@@ -0,0 +1,2 @@
+user default on #5b11618c2e44027877d0cd0921ed166b9f176f50587fc91e7534dd2946db77d6 ~* &* +@all
+user interwebz on #0b14d501a594442a01c6859541bcb3e8164d183d32937b851835442f69d5c94e ~* resetchannels -@all +monitor +@string +@hash +@geo +@bitmap +@set +@hyperloglog +@stream +@sortedset +@list +@keyspace -pfdebug +ping +command -brpop -move -pfselftest -restore -restore-asking -bzpopmax -blpop -keys -copy -sort  +info -xread +memory|usage  +client|id -swapdb +echo -blmove -brpoplpush -bzpopmin -flushdb -flushall -randomkey -xreadgroup -migrate +keys +time +ts.create +bf.add +bf.exists +bf.info +bf.insert +bf.loadchunk +bf.madd +bf.mexists +bf.reserve +bf.scandump +cf.add +cf.addnx +cf.count +cf.del +cf.exists +cf.info +cf.insert +cf.insertnx +cf.loadchunk +cf.mexists +cf.reserve +cf.scandump +cms.incrby +cms.info +cms.initbydim +cms.initbyprob +cms.merge +cms.query +json.arrappend +json.arrindex +json.arrinsert +json.arrlen +json.arrpop +json.arrtrim +json.clear +json.debug +json.debug|help +json.debug|memory +json.del +json.forget +json.get +json.mget +json.numincrby +json.nummultby +json.objkeys +json.objlen +json.resp +json.set +json.strappend +json.strlen +json.toggle +json.type +ts.add +ts.alter +ts.create -ts.createrule +ts.decrby +ts.del +ts.get +ts.incrby +ts.info +ts.madd +ts.range +ts.revrange +topk.add +topk.count +topk.incrby +topk.info +topk.list +topk.query +topk.reserve

docker/redis-stack/redis.conf

@@ -0,0 +1,11 @@
+maxmemory 4gb
+maxmemory-policy allkeys-lru
+
+save ""
+
+aclfile /etc/redis/interwebz.aclfile
+loadmodule /opt/redis-stack/lib/redisearch.so
+loadmodule /opt/redis-stack/lib/redisgraph.so
+loadmodule /opt/redis-stack/lib/redistimeseries.so
+loadmodule /opt/redis-stack/lib/rejson.so
+loadmodule /opt/redis-stack/lib/redisbloom.so

interwebz/__init__.py

@@ -29,6 +29,7 @@ def create_app(test_config=None):
     app.clients = {db['id']: NameSpacedRedis.from_url(
         db['url'], decode_responses=True) for db in app.config['DBS']}
     app.default_client = app.clients[app.config['DBS'][0]['id']]
+    app.stack_client = app.clients['stack']
 
     def reply(value: Any, error: bool) -> dict:
         """ API reply object. """
@@ -54,7 +55,10 @@ def post_command(dbid=None):
             return ''
 
         psession = PageSession()
-        client = app.default_client
+        if commands[0].split(' ')[0].lower() in app.default_client.commands:
+            client = app.default_client
+        else:
+            client = app.stack_client
         if dbid is not None:
             if dbid in app.clients:
                 client = app.clients[dbid]

interwebz/api.py

@@ -5,11 +5,17 @@
 
 from .pagesession import PageSession
 from .redis import NameSpacedRedis
+from math import log
+from math import log2
+from math import pow
 
 max_batch_size = 20
 max_arguments = 25
 max_argument_size = 256
+max_bits_allowed = 16000
+max_bytes_allowed = max_bits_allowed * 8
 
+ts_cmds = ['ts.create','ts.add','ts.alter','ts.incrby','ts.decrby']
 
 def reply(value: Any, error: bool) -> dict:
     return {
@@ -31,6 +37,7 @@ def sanitize_exceptions(argv: list) -> Any:
     # TODO: potential "attack" vectors: append, bitfield, sadd, zadd, xadd, hset, lpush/lmove*, sunionstore, zunionstore, ...
     cmd_name = argv[0].lower()
     argc = len(argv)
+    argv_lower = [x.lower()  for x in argv]
     if cmd_name == 'setbit' and argc == 4:
         try:
             offset = int(argv[2])
@@ -39,6 +46,27 @@ def sanitize_exceptions(argv: list) -> Any:
                 return f'offset too big - only up to {max_offset} bits allowed'
         except ValueError:
             pass  # Let the Redis server return a proper parsing error :)
+    elif cmd_name == 'bf.reserve' and argc >=4 :
+        try:
+            error_rate = float(argv[2])
+            capacity = float(argv[3])
+            return verify_bf(error_rate, capacity, argv, argv_lower, cmd_name)
+        except Exception as e:
+            raise e
+    elif cmd_name == 'bf.insert':
+        capacity_idx = argv_lower.index('capacity')
+        error_idx = argv_lower.index('error')
+        res = None
+        if(capacity_idx >= 1) and error_idx >= 1 and argc> max(capacity_idx, error_idx):
+            res = verify_bf(float(argv[error_idx+1]), float(argv[capacity_idx+1]), argv, argv_lower, cmd_name)
+    elif cmd_name == 'cms.initbydim' or cmd_name == 'cms.initbyprob':
+        return verify_cms(argv, cmd_name)
+    elif cmd_name == 'cf.reserve' or cmd_name == 'cf.insert' or cmd_name == 'cf.insertnx':
+        return verify_cf(argv_lower, cmd_name)
+    elif cmd_name == 'topk.reserve':
+        return verify_topk(argv)
+    elif cmd_name in ts_cmds:
+        return verify_ts_create(argv_lower)
     elif cmd_name == 'setrange' and argc == 4:
         try:
             offset = int(argv[2])
@@ -52,6 +80,79 @@ def sanitize_exceptions(argv: list) -> Any:
 
     return None
 
+def verify_ts_create(argv_lower) -> Any:    
+    illegal_arguments = []
+    if 'chunk_size' in argv_lower:
+        illegal_arguments.append('chunk_size')
+    if 'encoding' in argv_lower:
+        illegal_arguments.append('encoding')
+    if 'labels' in argv_lower:
+        illegal_arguments.append('labels')
+    if len(illegal_arguments) == 1:
+        return f'Argument "{illegal_arguments[0]}" is not'
+    elif len(illegal_arguments) > 0:
+        return f'Arguments {illegal_arguments} are not'
+
+def verify_topk(argv) -> Any:
+    k = int(argv[2])
+    width = 0
+    depth = 0
+    if len(argv) > 3:
+        width = int(argv[3])
+        depth = int(argv[4])
+
+    bytes_required = (k * 13 + width * depth * 8)
+    if bytes_required > max_bytes_allowed:
+        return f'TOPK.RESERVE requests more than allowed bytes, requested {bytes_required} only {max_bytes_allowed}'
+
+def verify_cf(argv_lower, cmd_name) -> Any:
+    capacity = 1024
+    bucket_size = 1
+    if cmd_name == 'cf.reserve':
+        if 'bucketsize' in argv_lower and len(argv_lower) > argv_lower.index('bucketsize')+1:
+            bucket_size = int(argv_lower[argv_lower.index('bucketsize')+1])
+        capacity = int(argv_lower[2])
+    elif cmd_name == 'cf.insert':
+        if 'capacity' in argv_lower and len(argv_lower) > argv_lower.index('capacity') + 1:
+            capacity = int(argv_lower[argv_lower.index('capacity') + 1])
+    if 'expansion' in argv_lower:
+        return 'Use of the EXPANSION argument is not'
+    num_bits_required = ceil((log2(1./((2.*bucket_size)/255))/.955) * capacity)
+    if num_bits_required > max_bits_allowed:
+        return f'{cmd_name} requests more thant allowed bits, requested {num_bits_required} only {max_bits_allowed}'
+
+def verify_cms(argv, cmd_name) -> Any:
+
+    width = 0
+    depth = 0
+    if cmd_name == 'cms.initbyprob':
+        width = ceil(2./float(argv[2]))
+        print(argv[3])
+        depth = ceil(log10(float(argv[3]))/log10(.5))
+    elif cmd_name == 'cms.initbydim':
+        width = int(argv[2])
+        depth = int(argv[3])
+    num_bits_required = width * depth * 64
+    print(num_bits_required)
+    if num_bits_required > max_bits_allowed:
+        return f'{cmd_name} requests more thant allowed bits, requested {num_bits_required} only {max_bits_allowed}'
+    return None
+
+
+def verify_bf(error_rate, capacity, argv, argv_lower, cmd_name) -> Any:
+    bits_per_item = -log2(error_rate)/(pow(log(2), 2))
+    num_bits_required = bits_per_item * capacity
+    if num_bits_required > max_bits_allowed:
+        return f'{cmd_name} asking for too much memory, {int(num_bits_required)} bits requested, only {max_bits_allowed} '
+    elif 'expansion' in argv_lower:
+        return 'Use of the EXPANSION argument is not'
+    elif not 'nonscaling' in argv_lower:
+        if 'items' in argv_lower:
+            items_idx = argv_lower.index('items')
+            argv.insert(items_idx, 'NONSCALING')
+        else:
+            argv.append('NONSCALING')
+    return None
 
 def verify_commands(commands: Any) -> Any:
     if type(commands) is not list:

interwebz/redis.py

@@ -141,38 +141,41 @@ def _key_spec_to_dict(response: list) -> dict:
     def _parse_command_response(self, response, **options):
         for command in response:
             command_spec = CommandSpec(command[1], command[2])
-            for s in command[8]:
-                key_spec = NameSpacedRedis._key_spec_to_dict(s) # spec start at possition 8
-                begin_type = key_spec['begin_search']['type']
-                begin_search = None
-                if begin_type == 'index':
-                    begin_search = BeginSearchIndex(key_spec['begin_search']['spec']['index'])
-                elif begin_type == 'keyword':
-                    keyword = key_spec['begin_search']['spec']['keyword']
-                    start_from = key_spec['begin_search']['spec']['startfrom']
-                    begin_search = BeginSearchKeyord(keyword, start_from)
-
-                if begin_search is None:
-                    continue
-
-                find_type = key_spec['find_keys']['type']
-                find_keys = None
-                if find_type == 'range':
-                    step = key_spec['find_keys']['spec']['keystep']
-                    lastkey = key_spec['find_keys']['spec']['lastkey']
-                    limit = key_spec['find_keys']['spec']['limit']
-                    find_keys = FindKeysRange(step, lastkey, limit)
-                elif find_type == 'keynum':
-                    step = key_spec['find_keys']['spec']['keystep']
-                    firstkey = key_spec['find_keys']['spec']['firstkey']
-                    keynumidx = key_spec['find_keys']['spec']['keynumidx']
-                    find_keys = FindKeysNum(step, firstkey, keynumidx)
-
-                if find_keys is None:
-                    continue
-
-                key_spec = KeySpec(begin_search, find_keys)
-                command_spec.add_key_spec(key_spec)
+            if len(command)>7:
+                for s in command[8]:
+                    key_spec = NameSpacedRedis._key_spec_to_dict(s) # spec start at possition 8
+                    begin_type = key_spec['begin_search']['type']
+                    begin_search = None
+                    if begin_type == 'index':
+                        begin_search = BeginSearchIndex(key_spec['begin_search']['spec']['index'])
+                    elif begin_type == 'keyword':
+                        keyword = key_spec['begin_search']['spec']['keyword']
+                        start_from = key_spec['begin_search']['spec']['startfrom']
+                        begin_search = BeginSearchKeyord(keyword, start_from)
+
+                    if begin_search is None:
+                        continue
+
+                    find_type = key_spec['find_keys']['type']
+                    find_keys = None
+                    if find_type == 'range':
+                        step = key_spec['find_keys']['spec']['keystep']
+                        lastkey = key_spec['find_keys']['spec']['lastkey']
+                        limit = key_spec['find_keys']['spec']['limit']
+                        find_keys = FindKeysRange(step, lastkey, limit)
+                    elif find_type == 'keynum':
+                        step = key_spec['find_keys']['spec']['keystep']
+                        firstkey = key_spec['find_keys']['spec']['firstkey']
+                        keynumidx = key_spec['find_keys']['spec']['keynumidx']
+                        find_keys = FindKeysNum(step, firstkey, keynumidx)
+
+                    if find_keys is None:
+                        continue
+
+                    key_spec = KeySpec(begin_search, find_keys)
+                    command_spec.add_key_spec(key_spec)
+            else:
+                command_spec.moveable_keys = True # this is pre Redis 7, the keyspec is incomplete so always scrutinize the command's keys
             self.commands[command[0]] = command_spec
 
         for c in self.commands.keys():
@@ -233,10 +236,13 @@ def execute_namespaced(self, session: PageSession, argv: list) -> Any:
             get_keys_args = argv.copy()
             get_keys_args.insert(0, 'GETKEYS')
             get_keys_args.insert(0, "COMMAND")
-            mapping = self.execute_command(*get_keys_args)
-            for arg in mapping:
-                idx = argv.index(arg)
-                argv[idx] = f'{session}:{arg}'
+            try:
+                mapping = self.execute_command(*get_keys_args)
+                for arg in mapping:
+                    idx = argv.index(arg)
+                    argv[idx] = f'{session}:{arg}'
+            except:
+                pass
         else:
             # Quickly check arity
             if cmd.arity > 0 and argc != cmd.arity:

sample_settings.json

@@ -9,6 +9,10 @@
         {
             "id": "default",
             "url": "redis://interwebz:password1@localhost:6379"
+        },
+        {
+            "id": "stack",
+            "url": "redis://interwebz:password1@localhost:6378"
         }
     ],
     "SECRET_KEY": "some_random_value",