Merge pull request #1866 from rtfd/indoc-subproject-search

Show subprojects in search results

readthedocs/core/static-src/core/js/doc-embed/search.js

@@ -21,6 +21,7 @@ function attach_elastic_search_query(data) {
         version = data.version,
         language = data.language || 'en',
         api_host = data.api_host,
+        subprojects = data.subprojects || {},
         canonical_url = data.canonical_url || "/";
 
     var query_override = function (query) {
@@ -46,6 +47,10 @@ function attach_elastic_search_query(data) {
                             highlight = hit.highlight;
 
                         item_url.href = canonical_url;
+                        if (fields.project != project) {
+                            var subproject_url = subprojects[fields.project];
+                            item_url.href = subproject_url;
+                        }
                         item_url += fields.path + DOCUMENTATION_OPTIONS.FILE_SUFFIX;
                         item_url.search = '?highlight=' + $.urlencode(query);
 

readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl

@@ -93,6 +93,9 @@ context = {
     'downloads': [ {% for key, val in downloads.items %}
     ("{{ key }}", "{{ val }}"),{% endfor %}
     ],
+    'subprojects': [ {% for slug, url in project.get_subproject_urls %}
+        ("{{ slug }}", "{{ url }}"),{% endfor %}
+    ],
     'slug': '{{ project.slug }}',
     'name': u'{{ project.name }}',
     'rtd_language': u'{{ project.language }}',

readthedocs/projects/models.py

@@ -359,6 +359,21 @@ def get_canonical_url(self):
         else:
             return self.get_docs_url()
 
+    def get_subproject_urls(self):
+        """List subproject URLs
+
+        This is used in search result linking
+        """
+        if getattr(settings, 'DONT_HIT_DB', True):
+            return [(proj['slug'], proj['canonical_url'])
+                    for proj in (
+                        apiv2.project(self.pk)
+                        .subprojects()
+                        .get()['subprojects'])]
+        else:
+            return [(proj.child.slug, proj.child.get_docs_url())
+                    for proj in self.subprojects.all()]
+
     def get_production_media_path(self, type_, version_slug, include_file=True):
         """
         This is used to see if these files exist so we can offer them for download.

readthedocs/restapi/utils.py

@@ -85,14 +85,16 @@ def delete_versions(project, version_data):
 
 def index_search_request(version, page_list, commit, project_scale, page_scale,
                          section=True, delete=True):
-    log_msg = ' '.join([page['path'] for page in page_list])
-    log.info("(Server Search) Indexing Pages: %s [%s]" % (
-        version.project.slug, log_msg))
+    """Update search indexes with build output JSON
+
+    In order to keep sub-projects all indexed on the same shard, indexes will be
+    updated using the parent project's slug as the routing value.
+    """
     project = version.project
-    page_obj = PageIndex()
-    section_obj = SectionIndex()
 
-    # tags = [tag.name for tag in project.tags.all()]
+    log_msg = ' '.join([page['path'] for page in page_list])
+    log.info("Updating search index: project=%s pages=[%s]",
+             project.slug, log_msg)
 
     project_obj = ProjectIndex()
     project_obj.index_document(data={
@@ -107,11 +109,17 @@ def index_search_request(version, page_list, commit, project_scale, page_scale,
         'weight': project_scale,
     })
 
+    page_obj = PageIndex()
+    section_obj = SectionIndex()
     index_list = []
     section_index_list = []
+    routes = [project.slug]
+    routes.extend([p.parent.slug for p in project.superprojects.all()])
     for page in page_list:
-        log.debug("(API Index) %s:%s" % (project.slug, page['path']))
-        page_id = hashlib.md5('%s-%s-%s' % (project.slug, version.slug, page['path'])).hexdigest()
+        log.debug("Indexing page: %s:%s" % (project.slug, page['path']))
+        page_id = (hashlib
+                   .md5('-'.join([project.slug, version.slug, page['path']]))
+                   .hexdigest())
         index_list.append({
             'id': page_id,
             'project': project.slug,
@@ -127,10 +135,10 @@ def index_search_request(version, page_list, commit, project_scale, page_scale,
         if section:
             for section in page['sections']:
                 section_index_list.append({
-                    'id': hashlib.md5(
-                        '%s-%s-%s-%s' % (project.slug, version.slug,
-                                         page['path'], section['id'])
-                    ).hexdigest(),
+                    'id': (hashlib
+                           .md5('-'.join([project.slug, version.slug,
+                                         page['path'], section['id']]))
+                           .hexdigest()),
                     'project': project.slug,
                     'version': version.slug,
                     'path': page['path'],
@@ -139,12 +147,15 @@ def index_search_request(version, page_list, commit, project_scale, page_scale,
                     'content': section['content'],
                     'weight': page_scale,
                 })
-            section_obj.bulk_index(section_index_list, parent=page_id, routing=project.slug)
+            for route in routes:
+                section_obj.bulk_index(section_index_list, parent=page_id,
+                                       routing=route)
 
-    page_obj.bulk_index(index_list, parent=project.slug)
+    for route in routes:
+        page_obj.bulk_index(index_list, parent=project.slug, routing=route)
 
     if delete:
-        log.info("(Server Search) Deleting files not in commit: %s" % commit)
+        log.info("Deleting files not in commit: %s", commit)
         # TODO: AK Make sure this works
         delete_query = {
             "query": {

readthedocs/restapi/views/search_views.py

@@ -6,6 +6,7 @@
 
 from readthedocs.builds.constants import LATEST
 from readthedocs.builds.models import Version
+from readthedocs.projects.models import Project
 from readthedocs.search.lib import search_file, search_project, search_section
 from readthedocs.restapi import utils
 
@@ -44,8 +45,8 @@ def search(request):
         return Response({'error': 'Need project and q'},
                         status=status.HTTP_400_BAD_REQUEST)
     log.debug("(API Search) %s", query)
-    results = search_file(request=request, project=project_slug,
-                          version=version_slug, query=query)
+    results = search_file(request=request, project_slug=project_slug,
+                          version_slug=version_slug, query=query)
     return Response({'results': results})
 
 

readthedocs/search/lib.py

@@ -1,7 +1,9 @@
-from readthedocs.builds.constants import LATEST
+from django.http import Http404
 
 from .indexes import PageIndex, ProjectIndex, SectionIndex
 
+from readthedocs.builds.constants import LATEST
+from readthedocs.projects.models import Project
 from readthedocs.search.signals import (before_project_search,
                                         before_file_search,
                                         before_section_search)
@@ -42,7 +44,17 @@ def search_project(request, query, language=None):
     return ProjectIndex().search(body)
 
 
-def search_file(request, query, project=None, version=LATEST, taxonomy=None):
+def search_file(request, query, project_slug=None, version_slug=LATEST, taxonomy=None):
+    """Search index for files matching query
+
+    Raises a 404 error on missing project
+
+    :param request: request instance
+    :param query: string to query for
+    :param project_slug: :py:cls:`Project` slug
+    :param version_slug: slug for :py:cls:`Project` version slug
+    :param taxonomy: taxonomy for search
+    """
     kwargs = {}
     body = {
         "query": {
@@ -93,17 +105,33 @@ def search_file(request, query, project=None, version=LATEST, taxonomy=None):
         "size": 50  # TODO: Support pagination.
     }
 
-    if project or version or taxonomy:
+    if project_slug or version_slug or taxonomy:
         final_filter = {"and": []}
 
-        if project:
-            final_filter['and'].append({'term': {'project': project}})
-
-            # Add routing to optimize search by hitting the right shard.
-            kwargs['routing'] = project
-
-        if version:
-            final_filter['and'].append({'term': {'version': version}})
+        if project_slug:
+            try:
+                project = (Project.objects
+                           .api(request.user)
+                           .get(slug=project_slug))
+                project_slugs = [project.slug]
+                project_slugs.extend(s.child.slug for s
+                                     in project.subprojects.all())
+                final_filter['and'].append({"terms": {"project": project_slugs}})
+
+                # Add routing to optimize search by hitting the right shard.
+                # This purposely doesn't apply routing if the project has more
+                # than one parent project.
+                if project.superprojects.exists():
+                    if project.superprojects.count() == 1:
+                        kwargs['routing'] = (project.superprojects.first()
+                                             .parent.slug)
+                else:
+                    kwargs['routing'] = project_slug
+            except Project.DoesNotExist:
+                return None
+
+        if version_slug:
+            final_filter['and'].append({'term': {'version': version_slug}})
 
         if taxonomy:
             final_filter['and'].append({'term': {'taxonomy': taxonomy}})
@@ -115,8 +143,7 @@ def search_file(request, query, project=None, version=LATEST, taxonomy=None):
 
     before_file_search.send(request=request, sender=PageIndex, body=body)
 
-    results = PageIndex().search(body, **kwargs)
-    return results
+    return PageIndex().search(body, **kwargs)
 
 
 def search_section(request, query, project_slug=None, version_slug=LATEST,

readthedocs/search/views.py

@@ -51,8 +51,8 @@ def elastic_search(request):
         if type == 'project':
             results = search_lib.search_project(request, query, language=language)
         elif type == 'file':
-            results = search_lib.search_file(request, query, project=project,
-                                             version=version,
+            results = search_lib.search_file(request, query, project_slug=project,
+                                             version_slug=version,
                                              taxonomy=taxonomy)
 
     if results:

readthedocs/templates/sphinx/layout.html

@@ -22,6 +22,7 @@
     project: "{{ slug }}",
     version: "{{ current_version }}",
     language: "{{ rtd_language }}",
+    subprojects: {},
     canonical_url: "{{ canonical_url }}",
     page: "{{ pagename }}",
     builder: "sphinx",
@@ -35,6 +36,9 @@
     api_host: "{{ api_host }}",
     commit: "{{ commit }}"
   };
+  {% for slug, url in subprojects %}
+  READTHEDOCS_DATA.subprojects["{{ slug }}"] = "{{ url }}";
+  {% endfor %}
   // Old variables
   var doc_version = "{{ current_version }}";
   var doc_slug = "{{ slug }}";