Skip to content

[bug 740169] ES search admin page updates #559

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
willkg wants to merge 1 commit into from
Closed

[bug 740169] ES search admin page updates #559

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions apps/search/admin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,13 @@
from django.shortcuts import render_to_response
from django.template import RequestContext

from waffle.models import Flag

from search.es_utils import (get_doctype_stats, get_indexes, delete_index,
ESTimeoutError, ESMaxRetryError,
ESIndexMissingException)
from search.models import Record
from search.tasks import (ES_REINDEX_PROGRESS, ES_WAFFLE_WHEN_DONE,
reindex_with_progress)
from search.tasks import ES_REINDEX_PROGRESS, reindex_with_progress
from sumo.urlresolvers import reverse


Expand Down Expand Up @@ -58,8 +59,7 @@ def search(request):

reindex_requested = 'reindex' in request.POST
if reindex_requested:
reindex_with_progress.delay(
waffle_when_done='waffle_when_done' in request.POST)
reindex_with_progress.delay()

delete_requested = 'delete_index' in request.POST
if delete_requested:
Expand Down Expand Up @@ -108,9 +108,12 @@ def search(request):

recent_records = reversed(Record.objects.order_by('starttime')[:20])

es_waffle_flag = Flag.objects.get(name=u'elasticsearch')

return render_to_response(
'search/admin/search.html',
{'title': 'Search',
'es_waffle_flag': es_waffle_flag,
'doctype_stats': stats,
'doctype_write_stats': write_stats,
'indexes': indexes,
Expand All @@ -122,11 +125,8 @@ def search(request):
# Dim the buttons even if the form loads before the task fires:
'progress': cache.get(ES_REINDEX_PROGRESS,
'0.001' if reindex_requested else ''),
'waffle_when_done':
request.POST.get('waffle_when_done') if reindex_requested else
cache.get(ES_WAFFLE_WHEN_DONE),
'progress_url': reverse('search.reindex_progress'),
'interval': settings.ES_REINDEX_PROGRESS_BAR_INTERVAL * 1000},
'progress_url': reverse('search.reindex_progress'),
'interval': settings.ES_REINDEX_PROGRESS_BAR_INTERVAL * 1000},
RequestContext(request, {}))


Expand Down
23 changes: 4 additions & 19 deletions apps/search/tasks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,23 +6,20 @@
from django.core.cache import cache

from celery.decorators import task
from waffle.models import Flag

from search.es_utils import es_reindex_with_progress


# This is present in memcached when reindexing is in progress and holds a float
# value between 0 and 100. When reindexing is complete (even if it crashes),
# the token is removed.
# This is present in memcached when reindexing is in progress and
# holds a float value between 0 and 100. When reindexing is complete
# (even if it crashes), the token is removed.
ES_REINDEX_PROGRESS = 'sumo:search:es_reindex_progress'
# Present iff ES should be waffled back on when indexing is complete:
ES_WAFFLE_WHEN_DONE = 'sumo:search:es_reindex_will_waffle'

log = logging.getLogger('k.task')


@task
def reindex_with_progress(waffle_when_done=False):
def reindex_with_progress():
"""Rebuild elasticsearch index while updating progress bar for admins."""
# Need to import Record here to prevent circular import
from search.models import Record
Expand All @@ -35,11 +32,6 @@ def reindex_with_progress(waffle_when_done=False):
# Init progress bar stuff:
cache.set(ES_REINDEX_PROGRESS, 0.001) # An iota so it tests true in
# the template
if waffle_when_done:
cache.set(ES_WAFFLE_WHEN_DONE, True)
else:
# Clear it in case there was already one there somehow:
cache.delete(ES_WAFFLE_WHEN_DONE)

# Reindex:
start = time()
Expand All @@ -52,17 +44,10 @@ def reindex_with_progress(waffle_when_done=False):
# to be understood by JS but makes me nervous:
cache.set(ES_REINDEX_PROGRESS, '%.5f' % ratio)

if cache.get(ES_WAFFLE_WHEN_DONE):
# Just go ahead and crash if the flag isn't there.
flag = Flag.objects.get(name='elasticsearch')
flag.everyone = True
flag.save()

rec.endtime = datetime.datetime.now()
rec.save()
finally:
cache.delete(ES_REINDEX_PROGRESS)
cache.delete(ES_WAFFLE_WHEN_DONE)


@task
Expand Down
80 changes: 54 additions & 26 deletions apps/search/templates/search/admin/search.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,8 @@ <h1>Elastic Search</h1>
{% endblock %}

{% block content %}
<section>
{% if es_error_message or delete_error_message %}
{% if es_error_message or delete_error_message %}
<section>
<h1>Status</h1>
{% if es_error_message %}
<p>
Expand All @@ -83,7 +83,9 @@ <h1>Status</h1>
delete_error_message: {{ delete_error_message }}
</p>
{% endif %}
{% else %}
</section>
{% else %}
<section>
<h1>Settings</h1>
<p>
Settings at the time this page was loaded:
Expand All @@ -94,7 +96,10 @@ <h1>Settings</h1>
<tr><th>ES_INDEXES</th><td>{{ settings.ES_INDEXES }}</td></tr>
<tr><th>ES_WRITE_INDEXES</th><td>{{ settings.ES_WRITE_INDEXES }}</td></tr>
</table>
<h1>Status</h1>
</section>

<section>
<h1>Index Status</h1>
<p>
All indexes available:
</p>
Expand Down Expand Up @@ -174,15 +179,19 @@ <h2>Write index ({{ write_index }})</h2>
</table>
{% endif %}
{% endif %}
{% endif %}
</section>
{% endif %}

<section>
<h1>Reindexing history</h1>
<table>
<thead>
<th>message</th>
<th>start time</th>
<th>end time</th>
<th>delta</th>
<tr>
<th>message</th>
<th>start time</th>
<th>end time</th>
<th>delta</th>
</tr>
</thead>
<tbody>
{% for record in recent_records %}
Expand All @@ -193,20 +202,51 @@ <h1>Reindexing history</h1>
<td>{{ record.delta }}</td>
</tr>
{% endfor %}
</body>
</tbody>
</table>
</section>

<section>
<h1>elasticsearch waffle flag</h1>
<table>
<thead>
<tr>
<th>everyone</th>
<th>percent</th>
<th>testing</th>
<th>superuser</th>
<th>staff</th>
</tr>
</thead>
<tbody>
<tr>
<td>{{ es_waffle_flag.everyone }}</td>
<td>{{ es_waffle_flag.percent }}</td>
<td>{{ es_waffle_flag.testing }}</td>
<td>{{ es_waffle_flag.superusers }}</td>
<td>{{ es_waffle_flag.staff }}</td>
</tr>
</tbody>
</table>
<p>
<a href="{% url admin:waffle_flag_change es_waffle_flag.id %}">change elasticsearch waffle flag</a>
</p>
</section>

<section>
<h1>Reindex</h1>
<p>
You should have to do this only when the search mapping changes or when
setting up the site for the first time.
You should have to do this only when the search mapping changes
or when setting up the site for the first time.
</p>
<p>
If the old index isn't compatible with the new code, you should waffle Sphinx on first.
If the old index isn't compatible with the new code, you should
waffle Sphinx on first.
</p>
{% if read_index == write_index %}
<p class="errornote">
WARNING! Read and write indexes are the same! Reindexing would be really bad!
WARNING! Read and write indexes are the same! Reindexing would
be really bad!
</p>
{% endif %}
<p>
Expand All @@ -220,18 +260,6 @@ <h1>Reindex</h1>
value="Reindex into {{ write_index }}"
{% if progress %}disabled="disabled"{% endif %}/>
</div>
<div>
<label>
<input type="checkbox"
{% if waffle_when_done %}checked="checked"{% endif %}
value="1"
name="waffle_when_done"
{% if progress %}disabled="disabled"{% endif %} />
<span{% if progress %} class="disabled"{% endif %}>
Waffle Sphinx off when finished
</span>
</label>
</div>
</form>
{% if progress %}
<p>
Expand Down