Bumps python to 3.10 and updates several packages

openstate · Dec 16, 2024 · f3af0e2 · f3af0e2
1 parent 64511f6
commit f3af0e2
Show file tree

Hide file tree

Showing 11 changed files with 62 additions and 78 deletions.
diff --git a/bin/backend.sh b/bin/backend.sh
@@ -2,16 +2,16 @@
 
 cd /opt/ori
 find . -name "*.pyc" -exec rm -f {}
-celery flower --app=ocd_backend.app:celery_app &
-#celery worker --app=ocd_backend.app:celery_app --attach --pidfile=/var/run/celery/%n.pid --loglevel=info --concurrency=8
+celery --app ocd_backend.app:celery_app flower &
+#celery --app=ocd_backend.app:celery_app worker --pidfile=/var/run/celery/%n.pid --loglevel=info --concurrency=8
 # nov 2024: using this now to start celery in development to get the logs. The "celery multi" command below may be faster
 # but for some reason does not write to the logs
-celery --app=ocd_backend.app:celery_app worker --loglevel=debug -Q transformers,enrichers --concurrency=8 --without-gossip --quiet
-# celery multi start 4 -Q transformers,enrichers -c8 -A ocd_backend.app:celery_app -l debug --pidfile=/var/run/celery/%n.pid
+celery --app=ocd_backend.app:celery_app --quiet worker --loglevel=debug -Q transformers,enrichers --concurrency=8 --without-gossip
+# celery -A ocd_backend.app:celery_app multi start 4 -Q transformers,enrichers -c8 -l debug --pidfile=/var/run/celery/%n.pid
 # while true
 #   do
 #     inotifywait -e modify,attrib,close_write,move,delete -r /opt/ori/ocd_backend \
 #       && date \
-#       && celery multi restart 4 -Q transformers,enrichers -c8 -A ocd_backend.app:celery_app -l debug --pidfile=/var/run/celery/%n.pid
+#       && celery -A ocd_backend.app:celery_app multi restart 4 -Q transformers,enrichers -c8 -l debug --pidfile=/var/run/celery/%n.pid
 #     sleep 10
 #   done
diff --git a/bin/worker.sh b/bin/worker.sh
@@ -1,3 +1,3 @@
 #!/bin/sh
 cd /opt/ori
-celery worker -A ocd_backend.app:celery_app -l debug
+celery -A ocd_backend.app:celery_app worker -l debug
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -41,7 +41,7 @@ services:
         condition: service_healthy
       postgres:
         condition: service_started
-    command: celery --app=ocd_backend.app:celery_app worker --loglevel=info -Q loaders --concurrency=1 --without-gossip --quiet
+    command: celery --app=ocd_backend.app:celery_app --quiet worker --loglevel=info -Q loaders --concurrency=1 --without-gossip
     #mem_limit: 1g
   redis:
     image: "redis:${REDIS_VERSION}"

diff --git a/manage.py b/manage.py
@@ -12,7 +12,6 @@
 import redis
 import click
 from click.core import Command
-from click.decorators import _make_command
 
 from elasticsearch.exceptions import RequestError
 from elasticsearch.helpers import reindex
@@ -33,22 +32,6 @@
 sys.path.insert(0, '/opt/ori')
 
 
-def command(name=None, cls=None, **attrs):
-    """
-    Wrapper for click Commands, to replace the click.Command docstring with the
-    docstring of the wrapped method (i.e. the methods defined below). This is
-    done to support the autodoc in Sphinx, and the correct display of docstrings
-    """
-    if cls is None:
-        cls = Command
-
-    def decorator(f):
-        r = _make_command(f, name, attrs, cls)
-        r.__doc__ = f.__doc__
-        return r
-    return decorator
-
-
 def _create_path(path):
     if not os.path.exists(path):
         click.secho('Creating path "%s"' % path, fg='green')
@@ -184,7 +167,7 @@ def developers():
     """Utilities for developers"""
 
 
-@command('put_template')
+@click.command('put_template')
 @click.option('--template_file', default='es_mappings/ori_template.json',
               type=click.File('rb'), help='Path to JSON file containing the template.')
 def es_put_template(template_file):
@@ -203,7 +186,7 @@ def es_put_template(template_file):
     es.indices.put_template('ori_template', template)
 
 
-@command('put_mapping')
+@click.command('put_mapping')
 @click.argument('index_name')
 @click.argument('mapping_file', type=click.File('rb'))
 def es_put_mapping(index_name, mapping_file):
@@ -222,7 +205,7 @@ def es_put_mapping(index_name, mapping_file):
     es.indices.put_mapping(index=index_name, body=mapping)
 
 
-@command('create_indexes')
+@click.command('create_indexes')
 @click.argument('mapping_dir', type=click.Path(exists=True, resolve_path=True))
 def create_indexes(mapping_dir):
     """
@@ -256,7 +239,7 @@ def create_indexes(mapping_dir):
             click.echo(error_msg)
 
 
-@command('delete_indexes')
+@click.command('delete_indexes')
 @click.option('--delete-template', is_flag=True, expose_value=True)
 def delete_indexes(delete_template):
     """
@@ -280,7 +263,7 @@ def delete_indexes(delete_template):
         es.indices.delete_template('ocd_template')
 
 
-@command('available_indices')
+@click.command('available_indices')
 def available_indices():
     """
     Shows a list of collections available at ``ELASTICSEARCH_HOST:ELASTICSEARCH_PORT``.
@@ -302,7 +285,7 @@ def available_indices():
     return available
 
 
-@command('copy')
+@click.command('copy')
 @click.argument('source_index')
 @click.argument('target_index')
 def es_copy_data(source_index, target_index):
@@ -323,7 +306,7 @@ def es_copy_data(source_index, target_index):
         }
     })
 
-@command('list_sources')
+@click.command('list_sources')
 @click.option('--sources_config', default=SOURCES_CONFIG_FILE)
 def extract_list_sources(sources_config):
     """
@@ -347,7 +330,7 @@ def extract_list_sources(sources_config):
         click.echo(' - %s' % source)
 
 
-@command('start')
+@click.command('start')
 @click.option('--sources_config', default=SOURCES_CONFIG_FILE)
 @click.argument('source_id')
 @click.option('--subitem', '-s', multiple=True)
@@ -417,7 +400,7 @@ def extract_start(source_id, subitem, entiteit, sources_config):
         click.echo('[%s] Processed pipelines: %s' % (source_id, ', '.join(selected_entities)))
 
 
-@command('load_redis')
+@click.command('load_redis')
 @click.argument('modus')
 @click.option('--source_path', default='*')
 @click.option('--sources_config', default=SOURCES_CONFIG_FILE)
@@ -446,7 +429,7 @@ def extract_load_redis(modus, source_path, sources_config):
             redis_client.set(redis_source, modus)
 
 
-@command('process')
+@click.command('process')
 @click.argument('modus')
 @click.option('--source_path', default='*')
 @click.option('--sources_config', default=SOURCES_CONFIG_FILE)
@@ -528,7 +511,7 @@ def extract_process(modus, source_path, sources_config):
             click.echo('Source %s in redis does not exist in available sources' % source)
 
 
-@command('monthly_check')
+@click.command('monthly_check')
 @click.option('--token')
 def es_monthly_check(token):
     result = get_recent_counts()
@@ -558,7 +541,7 @@ def es_monthly_check(token):
         )
         print(resp)
 
-@command('purge_dbs')
+@click.command('purge_dbs')
 @click.pass_context
 def developers_purge_dbs(ctx):
     """

diff --git a/ocd_backend/Dockerfile b/ocd_backend/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.9-alpine
+FROM python:3.10-alpine
 MAINTAINER Joep Meindertsma <[email protected]>
 
 COPY ocd_backend/requirements.txt /opt/ori/ocd_backend/requirements.txt
@@ -12,8 +12,8 @@ RUN apk add --update \
   libxml2-dev libxslt-dev poppler-dev jpeg-dev \
   inotify-tools libmagic
 RUN apk add postgresql-dev gcc python3-dev musl-dev
-RUN pip install --upgrade 'pip<24.1' \
-  && pip install cython==0.29.36 \
+RUN pip install --upgrade 'pip<24.3.1' \
+  && pip install cython==3.0.11 \
   && pip install --no-cache-dir -r /opt/ori/ocd_backend/requirements.txt \
   && pip uninstall -y cython \
   && cp /usr/share/zoneinfo/Europe/Amsterdam /etc/localtime \
@@ -43,4 +43,4 @@ VOLUME /opt/ori/data
 
 WORKDIR /opt/ori
 USER celery
-CMD celery --app=ocd_backend.app:celery_app worker --loglevel=info -Q transformers,enrichers --concurrency=3 --without-gossip --quiet
+CMD celery --app=ocd_backend.app:celery_app --quiet worker --loglevel=info -Q transformers,enrichers --concurrency=3 --without-gossip
diff --git a/ocd_backend/Dockerfile.dev b/ocd_backend/Dockerfile.dev
@@ -1,4 +1,4 @@
-FROM python:3.9-alpine
+FROM python:3.10-alpine
 MAINTAINER Joep Meindertsma <[email protected]>
 
 COPY ocd_backend/requirements.dev.txt /opt/ori/ocd_backend/requirements.txt
@@ -12,8 +12,8 @@ RUN apk add --update \
   libxml2-dev libxslt-dev poppler-dev jpeg-dev \
   inotify-tools libmagic
 RUN apk add postgresql-dev gcc python3-dev musl-dev
-RUN pip install --upgrade 'pip<24.1' \
-  && pip install cython==0.29.36 \
+RUN pip install --upgrade 'pip<24.3.1' \
+  && pip install cython==3.0.11 \
   && pip install --no-cache-dir -r /opt/ori/ocd_backend/requirements.txt \
   && pip uninstall -y cython \
   && cp /usr/share/zoneinfo/Europe/Amsterdam /etc/localtime \
@@ -43,4 +43,4 @@ VOLUME /opt/ori/data
 
 WORKDIR /opt/ori
 USER celery
-CMD celery --app=ocd_backend.app:celery_app worker --loglevel=info -Q transformers,enrichers --concurrency=3 --without-gossip --quiet
+CMD celery --app=ocd_backend.app:celery_app --quiet worker --loglevel=info -Q transformers,enrichers --concurrency=3 --without-gossip
diff --git a/ocd_backend/Dockerfile.test b/ocd_backend/Dockerfile.test
@@ -14,8 +14,8 @@ RUN apk add --update \
   libxml2-dev libxslt-dev poppler-dev jpeg-dev \
   inotify-tools libmagic nano
 RUN apk add postgresql-dev gcc python3-dev musl-dev
-RUN pip install --upgrade 'pip<24.1' \
-  && pip install cython==0.29.36 \
+RUN pip install --upgrade 'pip<24.3.1' \
+  && pip install cython==3.0.11 \
   && pip install --no-cache-dir -r /opt/ori/ocd_backend/requirements.txt \
   && pip uninstall -y cython \
   && cp /usr/share/zoneinfo/Europe/Amsterdam /etc/localtime \

diff --git a/ocd_backend/__init__.py b/ocd_backend/__init__.py
@@ -0,0 +1 @@
+from ocd_backend.app import celery_app
diff --git a/ocd_backend/requirements.dev.txt b/ocd_backend/requirements.dev.txt
@@ -1,28 +1,28 @@
-alembic==1.3.3
-celery[redis]==4.4.6
-redis==3.2.0
-click==2.1
+alembic==1.14.0
+celery[redis]==5.4.0
+redis==5.2.1
+click==8.1.2
 elasticsearch>=7.0.0,<8.0.0
-flower==0.9.3
+flower==2.0.1
 iso8601==0.1.12
-lxml==4.5.0
-kombu==4.6.10
+lxml==5.3.0
+kombu==5.4.2
 msgpack-python==0.4.2
 nose2==0.9.2
 pdftotext==2.1.4
 Pillow==9.3.0
 psycopg2==2.8.4
 # psycopg2-binary
 PyLD==1.0.5
-python-dateutil==2.7.3
+python-dateutil==2.9.0
 python-json-logger==0.1.11
 python-magic==0.4.15
 pyyaml==6.0.2
 rdflib==4.2.2
-requests[socks]==2.22.0
+requests[socks]==2.32.3
 sentry-sdk[celery]
 sqlalchemy==1.3.13
 sqlalchemy-utils==0.36.1
-urllib3==1.25.8
+urllib3==2.2.3
 translitcodec==0.7.0
 zeep==3.4.0
diff --git a/ocd_backend/requirements.txt b/ocd_backend/requirements.txt
@@ -1,28 +1,28 @@
-alembic==1.3.3
-celery[redis]==4.4.6
-redis==3.2.0
-click==2.1
+alembic==1.14.0
+celery[redis]==5.4.0
+redis==5.2.1
+click==8.1.2
 elasticsearch>=7.0.0,<8.0.0
-flower==0.9.3
+flower==2.0.1
 iso8601==0.1.12
-lxml==4.5.0
-kombu==4.6.10
+lxml==5.3.0
+kombu==5.4.2
 msgpack-python==0.4.2
 nose2==0.9.2
 pdftotext==2.1.4
 Pillow==9.3.0
 psycopg2==2.8.4
 # psycopg2-binary
 PyLD==1.0.5
-python-dateutil==2.7.3
+python-dateutil==2.9.0
 python-json-logger==0.1.11
 python-magic==0.4.15
 pyyaml==6.0.2
 rdflib==4.2.2
-requests==2.22.0
+requests==2.32.3
 sentry-sdk[celery]
 sqlalchemy==1.3.13
 sqlalchemy-utils==0.36.1
-urllib3==1.25.8
+urllib3==2.2.3
 translitcodec==0.7.0
 zeep==3.4.0
diff --git a/ocd_backend/settings.py b/ocd_backend/settings.py
@@ -40,22 +40,22 @@
 loaders_exchange = Exchange('loaders', type='direct')
 
 CELERY_CONFIG = {
-    'BROKER_URL': REDIS_URL,
-    'CELERY_ACCEPT_CONTENT': ['ocd_serializer'],
-    'CELERY_TASK_SERIALIZER': 'ocd_serializer',
-    'CELERY_RESULT_SERIALIZER': 'ocd_serializer',
-    'CELERY_RESULT_BACKEND': 'ocd_backend.result_backends:OCDRedisBackend+%s' % REDIS_URL,
-    'CELERY_MESSAGE_COMPRESSION': 'gzip',
-    'CELERYD_HIJACK_ROOT_LOGGER': False,
+    'broker_url': REDIS_URL,
+    'accept_content': ['ocd_serializer'],
+    'task_serializer': 'ocd_serializer',
+    'result_serializer': 'ocd_serializer',
+    'result_backend': 'ocd_backend.result_backends:OCDRedisBackend+%s' % REDIS_URL,
+    'result_compression': 'gzip',
+    'worker_hijack_root_logger': False,
     # ACKS_LATE prevents two tasks triggered at the same time to hang
     # https://wiredcraft.com/blog/3-gotchas-for-celery/
     'CELERY_TASK_ACKS_LATE': True,
-    'CELERYD_PREFETCH_MULTIPLIER': 1,
+    'worker_prefetch_multiplier': 1,
     # Expire results after 30 minutes; otherwise Redis will keep
     # claiming memory for a day
-    'CELERY_TASK_RESULT_EXPIRES': 1800,
-    'CELERYD_REDIRECT_STDOUTS_LEVEL': 'INFO',
-    'CELERY_ROUTES': {
+    'result_expires': 1800,
+    'worker_redirect_stdouts_level': 'INFO',
+    'task_routes': {
         'ocd_backend.transformers.*': {
             'queue': 'transformers',
             'routing_key': 'transformers',
@@ -82,7 +82,7 @@
             'priority': 0,
         },
     },
-    'CELERY_QUEUES': (
+    'task_queues': (
         Queue('transformers', transformers_exchange, routing_key='transformers'),
         Queue('enrichers', enrichers_exchange, routing_key='enrichers'),
         Queue('loaders', loaders_exchange, routing_key='loaders'),