final version with file log and cron-ready management function

Author: maiiku

apps/pinger/forms.py

@@ -29,6 +29,6 @@ def clean_file(self):
         '''
         _file = self.cleaned_data["file"]
         for (num, line) in enumerate(_file):
-            if len(line.split('|')) < 2:
+            if len(line.split('|')) < 2 or len(line.split('|')) > 3:
                 raise forms.ValidationError("Invalid file format in line %d: %s" % (num+1, line))
         return _file

apps/pinger/management/__init__.py

@@ -0,0 +1 @@
+__author__ = 'ubuntu'

apps/pinger/management/commands/__init__.py

@@ -0,0 +1 @@
+__author__ = 'ubuntu'

apps/pinger/management/commands/pinger.py

@@ -0,0 +1,20 @@
+from django.core.management.base import BaseCommand, CommandError
+from apps.pinger.ping import SendPing
+
+class Command(BaseCommand):
+    '''
+    Manage script to run with cron
+    for corn to work with venv, cd to project, choose right pyhon, eun command
+    cd /home/my/project && /home/my/virtual/bin/python /home/my/project/manage.py pinger > /var/log/pinger_cron.log 2>&1
+    '''
+    help = 'Checks urls added to database. intended to use with cron'
+
+    def handle(self, *args, **options):
+
+        pinger = SendPing()
+        try:
+            pinger.run()
+        except Exception, e:
+            raise CommandError('Exception caught: %s, %s' % (e.__doc__, e.message))
+
+        self.stdout.write('Successfully run pinger!')

apps/pinger/models.py

@@ -2,6 +2,9 @@
 from django.core.urlresolvers import reverse
 from django.db import models
 from datetime import datetime as dt
+from django.utils.timezone import utc
+from settings.common import LOGFILE
+
 
 class BaseModel(models.Model):
     '''
@@ -47,7 +50,7 @@ class PingHistory(BaseModel):
     haystack = models.ForeignKey('Haystack')
     http_code=models.IntegerField(default=0)
     time_ms = models.IntegerField(default=0)
-    phrase_found = models.BooleanField(default=False)
+    phrase_found = models.BooleanField(default=True)
 
     @property
     def is_error(self):
@@ -61,7 +64,18 @@ def save(self, *args, **kwargs):
         self.haystack.last_time_ms = self.time_ms
         self.haystack.last_checked = self.created
         if not self.is_error:
-            self.haystack.last_ok = dt.utcnow()
+            self.haystack.last_ok = dt.utcnow().replace(tzinfo=utc)
         else:
-            self.haystack.last_error = dt.utcnow()
-        self.haystack.save()
+            self.haystack.last_error = dt.utcnow().replace(tzinfo=utc)
+        self.haystack.save()
+
+        #append to log file
+        line =  '[{status}] {url} @ {check_time}: got {code} response in {ms}ms and requirement {phrase_found}\n'.format(
+            status='FAILED' if self.is_error else 'SUCCESS',
+            url=self.haystack.url,
+            check_time=self.created,
+            code=self.http_code,
+            ms=self.time_ms,
+            phrase_found='FAILED' if not self.phrase_found else 'SUCCESS' )
+        with open(LOGFILE, 'a+') as f:
+            f.write(line)

apps/pinger/ping.py

@@ -1,7 +1,10 @@
 # -*- coding: utf-8 -*-
+import logging
 import urllib2
 from apps.pinger.models import Haystack, PingHistory
 
+logger = logging.getLogger('pinger')
+
 
 class SendPing(object):
     CHUNK = 16 * 1024
@@ -33,26 +36,30 @@ def check_phrase(self, phrase, urldata):
 
     def ping_straw(self, straw):
         import time
-        error = False
+        error = True
         code = 0
 
         tstart = time.time()
 
         try:
             url_data, code = self.stream_data_from_url(straw.url)
+            logger.debug('url data: %s' % url_data)
+            error=False
         except:
             error = True
 
         tend = time.time()
         ms = tend-tstart
 
         if not error:
-            error = self.check_phrase(straw.search_phrase, url_data)
+            check_result = self.check_phrase(straw.search_phrase, url_data)
+        else:
+            check_result=False
 
         ping = PingHistory(
             haystack=straw,
             time_ms=ms,
-            phrase_found=error,
+            phrase_found=check_result,
             http_code=code,
         )
         ping.save()

apps/pinger/urls.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 from django.conf.urls import patterns, url
-from apps.pinger.views import HaystackUploadView, HaystackCreateView, HaystackListView, HaystackUpdateView, HaystackDeleteView, HaystackRunView
+from apps.pinger.views import HaystackUploadView, HaystackCreateView, HaystackListView, HaystackUpdateView, HaystackDeleteView, HaystackRunView, HaystackLogView
 
 urlpatterns = patterns('',
     url(r'upload/$', HaystackUploadView.as_view(), name='hay_upload'),
@@ -9,6 +9,7 @@
     url(r'list/(?P<pk>\d+)/$', HaystackUpdateView.as_view(), name='hay_update'),
     url(r'list/(?P<pk>\d+)/delete/$', HaystackDeleteView.as_view(), name='hay_delete'),
     url(r'run/$', HaystackRunView.as_view(), name='hay_run'),
+    url(r'log/(?P<num>\d+)/$$', HaystackLogView.as_view(), name='hay_log'),
 
     url(r'$', HaystackListView.as_view(), name='home'),
 )

apps/pinger/views.py

@@ -1,11 +1,13 @@
 # -*- coding: utf-8 -*-
-from django.http import HttpResponseRedirect
+from django.http import HttpResponseRedirect, HttpResponse
 from django.views.generic import CreateView, ListView, UpdateView, DeleteView
 from django.views.generic.base import TemplateResponseMixin, ContextMixin, View
 from apps.pinger.forms import UploadFileForm, HaystackCreateModelForm
 from apps.pinger.models import Haystack
 from django.core.urlresolvers import reverse_lazy
 from apps.util.mixins import MessageMixin
+from settings.common import LOGFILE
+
 
 class HaystackUploadView(TemplateResponseMixin, ContextMixin, View):
     form_class = UploadFileForm
@@ -40,26 +42,31 @@ def line_to_haystack(self, line):
         )
         straw.save()
 
+
 class HaystackListView(ListView):
     model = Haystack
 
+
 class HaystackCreateView(MessageMixin, CreateView):
     model = Haystack
     form_class = HaystackCreateModelForm
     success_message='Added new item!'
     success_url = reverse_lazy('hay_list')
 
+
 class HaystackUpdateView(MessageMixin, UpdateView):
     model = Haystack
     form_class = HaystackCreateModelForm
     success_message='Data was successfully edited'
     success_url = reverse_lazy('hay_list')
 
+
 class HaystackDeleteView(MessageMixin, DeleteView):
     model = Haystack
     success_message='Link was removed from monitoring system.'
     success_url = reverse_lazy('hay_list')
 
+
 class HaystackRunView(View):
     success_url = reverse_lazy('hay_list')
 
@@ -69,3 +76,18 @@ def get(self, request, *args, **kwargs):
         pinger.run()
         return HttpResponseRedirect(self.success_url)
 
+
+class HaystackLogView(View):
+
+    def get(self, request, *args, **kwargs):
+        num = int(kwargs.pop('num') or 5)
+
+        from apps.util.util import tail
+
+        with open(LOGFILE, 'r') as f:
+            data, more = tail(f, num)
+            if more:
+                data = 'Showing last %d lines. Full og file avialbale at: %s\n\n%s' % (num, LOGFILE, data)
+        response = HttpResponse(data, content_type='text/plain')
+
+        return response

apps/util/util.py

@@ -0,0 +1,21 @@
+def tail(f, n, offset=None):
+    """Reads a n lines from f with an offset of offset lines.  The return
+    value is a tuple in the form ``(lines, has_more)`` where `has_more` is
+    an indicator that is `True` if there are more lines in the file.
+    """
+    avg_line_length = 74
+    to_read = n + (offset or 0)
+
+    while 1:
+        try:
+            f.seek(-(avg_line_length * to_read), 2)
+        except IOError:
+            # apparently file is smaller than what we want
+            # to step back, go to the beginning instead
+            f.seek(0)
+        pos = f.tell()
+        lines = f.read().splitlines()
+        if len(lines) >= to_read or pos == 0:
+            return '\n'.join(lines[-to_read:offset and -offset or None]), \
+                   len(lines) > to_read or pos > 0
+        avg_line_length *= 1.3