feat: [FC-86] implement multivalue faceted search using Meilisearch and Elasticsearch

.github/workflows/ci.yml

@@ -17,15 +17,21 @@ jobs:
         python-version: ['3.11', '3.12']
         toxenv: [django42, django52, quality]
 
+    env:
+      MEILISEARCH_URL: http://localhost:7700
+
     steps:
     - uses: actions/checkout@v2
     - name: setup python
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
 
+    - name: Start Meilisearch
+      run: make meili-up
+
     - name: Start Elasticsearch
-      run: make test.start_elasticsearch
+      run: make elastic-up
 
     - name: Install pip
       run: pip install -r requirements/pip.txt

Makefile

@@ -59,3 +59,34 @@ test: test_with_es ## run tests and generate coverage report
 install-local: ## installs your local edx-search into the LMS and CMS python virtualenvs
 	docker exec -t edx.devstack.lms bash -c '. /edx/app/edxapp/venvs/edxapp/bin/activate && cd /edx/app/edxapp/edx-platform && pip uninstall -y edx-search && pip install -e /edx/src/edx-search && pip freeze | grep edx-search'
 	docker exec -t edx.devstack.cms bash -c '. /edx/app/edxapp/venvs/edxapp/bin/activate && cd /edx/app/edxapp/edx-platform && pip uninstall -y edx-search && pip install -e /edx/src/edx-search && pip freeze | grep edx-search'
+
+test-all: create-test-network meili-up elastic-up
+	@MEILISEARCH_MASTER_KEY=test_master_key python manage.py test || true
+	@$(MAKE) meili-down
+	@$(MAKE) elastic-down
+
+
+meili-up: create-test-network
+	@echo "Starting Meilisearch..."
+	@docker compose up -d test_meilisearch
+	@echo "Waiting for Meilisearch to be healthy..."
+	@timeout 15 bash -c \
+    	'until curl -sf http://localhost:7700/health > /dev/null; do echo "Waiting..."; sleep 1; done'
+
+meili-down:
+	@echo "Shutting down Meilisearch..."
+	@docker compose down test_meilisearch
+
+
+elastic-up: create-test-network
+	@echo "Starting Elasticsearch..."
+	@docker compose up -d test_elasticsearch
+	@echo "Waiting for Elasticsearch to be healthy..."
+	@timeout 30 bash -c 'until curl -s http://localhost:9200/_cluster/health | grep -q "status"; do echo "Waiting..."; sleep 2; done'
+
+elastic-down:
+	@echo "Shutting down Elasticsearch..."
+	docker compose down test_elasticsearch
+
+create-test-network:
+	docker network inspect test_network >/dev/null 2>&1 || docker network create --driver bridge test_network

docker-compose.yml

@@ -1,3 +1,5 @@
+version: '3.8'
+
 services:
 
   test_elasticsearch:
@@ -18,7 +20,31 @@ services:
       - data01:/usr/share/elasticsearch/data
     ports:
       - "9200:9200"
+    networks:
+      - test_network
+
+  test_meilisearch:
+    # Keep in sync with DOCKER_IMAGE_MEILISEARCH
+    # in https://github.com/overhangio/tutor/blob/main/tutor/templates/config/defaults.yml
+    image: getmeili/meilisearch:v1.8.4
+    ports:
+      - "7700:7700"
+    networks:
+      - test_network
+    environment:
+      MEILISEARCH_MASTER_KEY: test_master_key
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:7700/health"]
+      interval: 2s
+      timeout: 1s
+      retries: 10
 
 volumes:
   data01:
     driver: local
+
+networks:
+  test_network:
+    name: test_network
+    driver: bridge
+    external: true

edxsearch/settings.py

@@ -26,7 +26,7 @@
 # This is just a container for running tests
 DEBUG = True
 
-ALLOWED_HOSTS = []
+ALLOWED_HOSTS = ['testserver', 'localhost', '127.0.0.1']
 
 TEMPLATES = [
     {
@@ -128,3 +128,14 @@
         }
     }
 }
+
+MEILISEARCH_API_KEY = os.environ.get("MEILISEARCH_MASTER_KEY", "test_master_key")
+MEILISEARCH_URL = os.environ.get("MEILISEARCH_URL", "http://localhost:7700")
+
+ELASTIC_SEARCH_CONFIG = [
+    {
+        'use_ssl': False,
+        'host': 'localhost',
+        'port': 9200
+    }
+]

search/api.py

@@ -129,6 +129,7 @@ def course_discovery_search(
     from_=0,
     field_dictionary=None,
     enable_course_sorting_by_start_date=False,
+    is_multivalue=False,  # Determines whether to use single-faceted or multi-faceted search
 ):
     """
     Course Discovery activities against the search engine index of course details
@@ -166,6 +167,7 @@ def course_discovery_search(
         exclude_dictionary=exclude_dictionary,
         aggregation_terms=course_discovery_aggregations(),
         sort_by=sort_by,
+        is_multivalue=is_multivalue,
     )
 
     return results

search/elastic.py

@@ -23,7 +23,7 @@
 RESERVED_CHARACTERS = "+=><!(){}[]^~*:\\/&|?"
 
 
-def _translate_hits(es_response):
+def _translate_hits(es_response, aggregation_terms, is_multivalue=False):
     """
     Provide result set in our desired format from elasticsearch results.
 
@@ -99,7 +99,7 @@ def translate_result(result):
 
     def translate_agg_bucket(bucket, agg_result):
         """
-        Any conversion from ES aggregations result into our search engine syntax
+        Convert ES aggregation result following our search engine syntax.
 
         agg_result argument needs for getting total number of
         documents per bucket.
@@ -109,19 +109,29 @@ def translate_agg_bucket(bucket, agg_result):
         :return: dict
         """
         agg_item = agg_result[bucket]
-        terms = {
-            bucket["key"]: bucket["doc_count"]
-            for bucket in agg_item["buckets"]
-        }
-        total_docs = (
-            agg_result[_get_total_doc_key(bucket)]["value"]
-            + agg_item["sum_other_doc_count"]
-            + agg_item["doc_count_error_upper_bound"]
-        )
+
+        if is_multivalue:
+            values_agg = agg_item["values"]
+            terms = {
+                bucket["key"]: bucket["doc_count"]
+                for bucket in values_agg["buckets"]
+            }
+            total_docs = sum(terms.values())
+        else:
+            terms = {
+                bucket["key"]: bucket["doc_count"]
+                for bucket in agg_item["buckets"]
+            }
+            total_docs = (
+                agg_result[_get_total_doc_key(bucket)]["value"]
+                + agg_item["sum_other_doc_count"]
+                + agg_item["doc_count_error_upper_bound"]
+            )
+
         return {
             "terms": terms,
             "total": total_docs,
-            "other": agg_item["sum_other_doc_count"],
+            "other": agg_item.get("sum_other_doc_count", 0),
         }
 
     results = list(map(translate_result, es_response["hits"]["hits"]))
@@ -132,11 +142,19 @@ def translate_agg_bucket(bucket, agg_result):
         "results": results,
     }
     if "aggregations" in es_response:
-        response["aggs"] = {
-            bucket: translate_agg_bucket(bucket, es_response["aggregations"])
-            for bucket in es_response["aggregations"]
-            if "total_" not in bucket
-        }
+        if is_multivalue:
+            global_aggs = es_response["aggregations"].get("global_aggs", {})
+            response["aggs"] = {
+                facet: translate_agg_bucket(facet, global_aggs)
+                for facet in global_aggs
+                if facet in aggregation_terms
+            }
+        else:
+            response["aggs"] = {
+                bucket: translate_agg_bucket(bucket, es_response["aggregations"])
+                for bucket in es_response["aggregations"]
+                if "total_" not in bucket
+            }
 
     return response
 
@@ -241,6 +259,51 @@ def _process_aggregation_terms(aggregation_terms):
     return elastic_aggs
 
 
+def _process_multivalue_aggregations(aggregation_terms: dict, field_dictionary: dict) -> dict:
+    """
+    Compute the aggregation requests that we'll send with the query when searching with multi-value faceting.
+    Typical shape of the args:
+        aggregation_terms={'language': {}, 'modes': {}, 'org': {}}
+        field_dictionary={'enrollment_start': <search.utils.DateRange object at ...>,
+                          'language': ['en', 'fr']}
+    """
+    aggs = {}
+    for facet_field, options in aggregation_terms.items():
+        filters_excluding_facet = {
+            field: value for field, value in field_dictionary.items()
+            if field != facet_field
+        }
+        filter_clauses = [
+            _get_filter_field(field, value)
+            for field, value in filters_excluding_facet.items()
+            if value
+        ]
+        facet_filter = {
+            "bool": {
+                "must": filter_clauses
+            }
+        } if filter_clauses else {"match_all": {}}
+
+        aggs[facet_field] = {
+            "filter": facet_filter,
+            "aggs": {
+                "values": {
+                    "terms": {
+                        "field": facet_field,
+                        **options
+                    }
+                }
+            }
+        }
+
+    return {
+        "global_aggs": {
+            "global": {},
+            "aggs": aggs
+        }
+    }
+
+
 class ElasticSearchEngine(SearchEngine):
     """
     ElasticSearch implementation of SearchEngine abstraction
@@ -654,8 +717,13 @@ def search(self,
                 }
 
         body = {"query": query}
+
+        is_multivalue = kwargs.pop("is_multivalue", False)
         if aggregation_terms:
-            body["aggs"] = _process_aggregation_terms(aggregation_terms)
+            if is_multivalue:
+                body["aggs"] = _process_multivalue_aggregations(aggregation_terms, field_dictionary)
+            else:
+                body["aggs"] = _process_aggregation_terms(aggregation_terms)
         if sort_by:
             body["sort"] = self._transform_sort_by(sort_by)
 
@@ -668,7 +736,7 @@ def search(self,
             log.exception("error while searching index - %r", ex)
             raise
 
-        return _translate_hits(es_response)
+        return _translate_hits(es_response, aggregation_terms, is_multivalue)
 
     def _transform_sort_by(self, fields: list[SortField]):
         """