New docker file

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:2.7.18-slim-stretch
+FROM debian:11-slim
 
 ENV PYTHONDONTWRITEBYTECODE=1
 
@@ -12,13 +12,14 @@ RUN apt-get update && apt-get install -y \
         gnupg2                           \
     && rm -rf /var/lib/apt/lists/*
 RUN cat /etc/pki/gpg/GPG-KEY-nodesource | apt-key add -
-ADD docker/debian-stretch-nodesource-repo /etc/apt/sources.list.d/nodesource.list
-ADD docker/debian-stretch-backports-repo /etc/apt/sources.list.d/backports.list
+ADD docker/debian-bullseye-nodesource-repo /etc/apt/sources.list.d/nodesource.list
+ADD docker/debian-bullseye-backports-repo /etc/apt/sources.list.d/backports.list
 
 RUN apt-get update && apt-get install -y \
         # General (dev-) dependencies
         bash-completion \
         build-essential \
+        cmake \
         curl \
         libjpeg-dev \
         libsasl2-dev \
@@ -28,14 +29,14 @@ RUN apt-get update && apt-get install -y \
         zlib1g-dev \
         libffi-dev \
         libssl-dev \
-        python-dev \
-        python-pip \
         nodejs \
         # Git, because we're using git-checkout dependencies
         git \
         # Dependencies for mysql-python
-        mysql-client \
-        default-libmysqlclient-dev \
+        mariadb-server \
+        mariadb-client \
+        libmariadb-dev \
+        libmariadb-dev-compat \
         swig \
         gettext \
         # Use rsvg-convert to render our static theme previews
@@ -44,16 +45,50 @@ RUN apt-get update && apt-get install -y \
         pngcrush \
         # our makefile and ui-tests require uuid to be installed
         uuid \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN apt-get update && apt-get -t stretch-backports install -y \
-        # For git-based files storage backend
-        libgit2-dev \
+        # Dependencies \
+        memcached \
+        nginx \
+        #elasticsearch \
+        redis-server \
+        rabbitmq-server \
+        npm \
+        wget \
     && rm -rf /var/lib/apt/lists/*
 
 # Compile required locale
 RUN localedef -i en_US -f UTF-8 en_US.UTF-8
 
+# Build libgit2-0.27
+RUN mkdir -p /build/libgit2
+RUN wget -P /build/libgit2 https://github.com/libgit2/libgit2/archive/refs/tags/v0.27.4.tar.gz
+RUN cd /build/libgit2 \
+    && tar -xf v0.27.4.tar.gz  \
+    && cd libgit2-0.27.4  \
+    && mkdir build \
+    && cd build \
+    && cmake .. \
+    && cmake --build . --target install
+
+# Build Python 2....
+RUN mkdir -p /build/python2
+RUN wget -P /build/python2 https://www.python.org/ftp/python/2.7.18/Python-2.7.18.tgz
+RUN cd /build/python2  \
+    && tar -xf Python-2.7.18.tgz  \
+    && cd Python-2.7.18  \
+    && ./configure --enable-optimizations \
+    && make -j ${nproc} . \
+    && make install
+
+ENV PYTHON_PIP_VERSION=20.0.2
+ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/d59197a3c169cef378a22428a3fa99d33e080a5d/get-pip.py
+ENV PYTHON_GET_PIP_SHA256=421ac1d44c0cf9730a088e337867d974b91bdce4ea2636099275071878cc189e
+ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+# Test it out!
+RUN python --version
+RUN python -m ensurepip --upgrade
+RUN python -m pip install --upgrade pip
+
 # Set the locale. This is mainly so that tests can write non-ascii files to
 # disk.
 ENV LANG en_US.UTF-8
@@ -90,4 +125,17 @@ ENV CLEANCSS_BIN /deps/node_modules/.bin/cleancss
 ENV LESS_BIN /deps/node_modules/.bin/lessc
 ENV UGLIFY_BIN /deps/node_modules/.bin/uglifyjs
 ENV ADDONS_LINTER_BIN /deps/node_modules/.bin/addons-linter
-RUN npm cache clean -f && npm install -g n && n 14.21
+RUN npm cache clean -f && npm install -g n && /deps/bin/n 14.21
+
+# Add our nginx config
+ADD docker/etc/nginx/sites-available/atn.conf /etc/nginx/sites-available/atn.conf
+
+# Add our mariadb config
+ADD docker/etc/mysql/my.cnf /etc/mysql/my.cnf
+ADD docker/etc/mysql/mariadb.conf.d/99-remote.cnf /etc/mysql/mariadb.conf.d/99-remote.cnf
+
+# Add our rabbitmq config
+ADD docker/etc/rabbitmq/rabbitmq.conf /etc/rabbitmq/rabbitmq.conf
+
+# Add our redis config
+ADD docker/etc/redis/redis.conf /etc/redis/redis.conf

Makefile-docker

@@ -89,11 +89,14 @@ populate_data:
 	python manage.py reindex --wipe --force --noinput
 	python manage.py generate_addons --app firefox $(NUM_ADDONS)
 	python manage.py generate_addons --app android $(NUM_ADDONS)
-	python manage.py generate_themes $(NUM_THEMES)
+	python manage.py generate_addons --app thunderbird $(NUM_ADDONS)
+	python manage.py generate_themes --app firefox $(NUM_THEMES)
+	python manage.py generate_themes --app thunderbird $(NUM_THEMES)
 	# These add-ons are specifically useful for the addons-frontend
 	# homepage. You may have to re-run this, in case the data there
 	# changes.
-	python manage.py generate_default_addons_for_frontend
+	# Stubbed out, uses pyfxa which is not pulled in.
+	#python manage.py generate_default_addons_for_frontend
 
 	# Now that addons have been generated, reindex.
 	python manage.py reindex --force --noinput

docker-compose.yml

@@ -1,12 +1,10 @@
-version: "2.3"
-
 x-env-mapping: &env
   environment:
-    - CELERY_BROKER_URL=amqp://olympia:olympia@rabbitmq/olympia
-    - CELERY_RESULT_BACKEND=redis://redis:6379/1
-    - DATABASES_DEFAULT_URL=mysql://root:@mysqld/olympia
+    - CELERY_BROKER_URL=amqp://olympia:olympia@services/olympia
+    - CELERY_RESULT_BACKEND=redis://services:6379/1
+    - DATABASES_DEFAULT_URL=mysql://root:docker@services:3306/olympia
     - ELASTICSEARCH_LOCATION=elasticsearch:9200
-    - MEMCACHE_LOCATION=memcached:11211
+    - MEMCACHE_LOCATION=services:11211
     - MYSQL_DATABASE=olympia
     - MYSQL_ROOT_PASSWORD=docker
     - OLYMPIA_SITE_URL=http://olympia.test
@@ -18,36 +16,42 @@ x-env-mapping: &env
 services:
   worker: &worker
     <<: *env
-    image: addons/addons-server
+    build: .
     command: supervisord -n -c /code/docker/supervisor-celery.conf
     entrypoint: ./scripts/start-docker.sh
     volumes:
       - .:/code
-    extra_hosts:
-     - "olympia.test:127.0.0.1"
 
   web:
     <<: *worker
     command: supervisord -n -c /code/docker/supervisor.conf
+    extra_hosts:
+      - "olympia.test:127.0.0.1"
+    ports:
+      - 8000:8000
 
-  nginx:
-    image: addons/addons-nginx
+  services:
+    <<: *worker
+    command: /code/scripts/start-docker-services.sh
+    entrypoint: '/bin/bash'
     volumes:
-      - ./static:/srv/static
-      - ./site-static:/srv/site-static
-      - ./user-media/:/srv/user-media
-
-  memcached:
-    image: memcached:1.4
-
-  mysqld:
-    image: mysql:5.7
-    environment:
-      - MYSQL_ALLOW_EMPTY_PASSWORD=yes
-      - MYSQL_DATABASE=olympia
+      - 'mariadbvolume:/var/lib/mysql'
+      - .:/code
+    expose:
+      - 6379:6379   # Redis
+      - 11211:11211 # Memcache
+    ports:
+      - 3306:3306   # Mariadb
+      - 80:80       # Nginx
 
   elasticsearch:
+    platform: linux/amd64
     image: docker.elastic.co/elasticsearch/elasticsearch:5.4.1
+    ulimits:
+        nproc: 65535
+        nofile:
+            soft: 20000
+            hard: 40000
     environment:
       # Disable all xpack related features to avoid unrelated logging
       # in docker logs. https://github.com/mozilla/addons-server/issues/8887
@@ -59,48 +63,9 @@ services:
       - xpack.watcher.enabled=false
       - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      # M1 mac emulation doesn't support this feature
+      - bootstrap.system_call_filter=false
     mem_limit: 2g
 
-  redis:
-    image: redis:2.8
-
-  rabbitmq:
-    image: rabbitmq:3.5
-    hostname: olympia
-    expose:
-      - "5672"
-    environment:
-      - RABBITMQ_DEFAULT_USER=olympia
-      - RABBITMQ_DEFAULT_PASS=olympia
-      - RABBITMQ_DEFAULT_VHOST=olympia
-
-  autograph:
-    image: mozilla/autograph:2.3.0
-
-  selenium-firefox:
-    <<: *env
-    image: b4handjr/selenium-firefox
-    volumes:
-      - .:/code
-    expose:
-      - "4444"
-    ports:
-      - "5900"
-    shm_size: 2g
-    links:
-      - "addons-frontend:olympia-frontend.test"
-      - "nginx:olympia.test"
-
-  addons-frontend:
-    <<: *env
-    environment:
-      - HOSTNAME=uitests
-      - WEBPACK_SERVER_HOST=olympia-frontend.test
-      - FXA_CONFIG=default
-    image: addons/addons-frontend
-    ports:
-      - "3000:3000"
-      - "3001:3001"
-    command: yarn amo:ui-tests
-    links:
-      - "nginx:olympia.test"
+volumes:
+  mariadbvolume:

docker/debian-bullseye-backports-repo

@@ -0,0 +1 @@
+deb http://deb.debian.org/debian bullseye-backports main

docker/debian-bullseye-elasticsearch-repo

@@ -0,0 +1 @@
+deb https://artifacts.elastic.co/packages/7.x/apt stable main

docker/debian-bullseye-nodesource-repo

@@ -0,0 +1,2 @@
+deb https://deb.nodesource.com/node_10.x bullseye main
+deb-src https://deb.nodesource.com/node_10.x bullseye main

docker/etc/elasticsearch/jvm.options.d/99-mem-limit

@@ -0,0 +1,2 @@
+-Xms512m
+-Xmx512m

docker/etc/mysql/mariadb.conf.d/99-remote.cnf

@@ -0,0 +1,2 @@
+[mysqld]
+bind-address=::

docker/etc/mysql/my.cnf

@@ -0,0 +1,29 @@
+# The MariaDB configuration file
+#
+# The MariaDB/MySQL tools read configuration files in the following order:
+# 0. "/etc/mysql/my.cnf" symlinks to this file, reason why all the rest is read.
+# 1. "/etc/mysql/mariadb.cnf" (this file) to set global defaults,
+# 2. "/etc/mysql/conf.d/*.cnf" to set global options.
+# 3. "/etc/mysql/mariadb.conf.d/*.cnf" to set MariaDB-only options.
+# 4. "~/.my.cnf" to set user-specific options.
+#
+# If the same option is defined multiple times, the last one will apply.
+#
+# One can use all long options that the program supports.
+# Run program with --help to get a list of available options and with
+# --print-defaults to see which it would actually understand and use.
+#
+# If you are new to MariaDB, check out https://mariadb.com/kb/en/basic-mariadb-articles/
+
+#
+# This group is read both by the client and the server
+# use it for options that affect everything
+#
+[client-server]
+# Port or socket location where to connect
+port = 3306
+# socket = /run/mysqld/mysqld.sock
+
+# Import all .cnf files from configuration directory
+!includedir /etc/mysql/conf.d/
+!includedir /etc/mysql/mariadb.conf.d/