initial import

Author: Buanzo

.gitignore

@@ -0,0 +1,6 @@
+*~
+*pyc
+DEADJOE
+tests/*
+.#*
+requirements.txt.old

README.md

@@ -0,0 +1,83 @@
+Web Monitoring
+--------------
+
+Webmon executes plugins against assets (URLs, originally).
+
+The API will contain groups of calls, divided in asset management,
+statistics gathering and execution.
+
+Asset Management involves URL management, adding, modifying and removing
+items (i.e "https://somesite.net, http://some.other.server/somespecificURI,
+etc").
+
+Statistics Gathering will provide statistical information about said assets.
+
+Execution will provide the necessary means to push jobs to the work queue.
+
+Storage is implemented in Redis
+
+
+Work Queue
+----------
+
+The Jobs API (A Ronald Class) will send jobs to a Messaging Queue (zeromq). 
+This Queue will be read by the Dispatcher, which is a standalone module. 
+
+
+Statistics
+----------
+
+The statistics API will provide information about assets. Information will
+come from the different plugins that the Executor enables. All plugins will
+be executed for each Asset.
+
+
+Executor Module (standalone)
+----------------------------
+
+The Executor Module will be a Task Worker, using the standard zeromq
+Streamer: http://learning-0mq-with-pyzmq.readthedocs.io/en/latest/pyzmq/devices/streamer.html
+
+Dispatcher Module (standalone)
+------------------------------
+
+The Dispatcher Module will act as a broker. The Jobs API class will use zeromq
+to communicate with the Dispatcher, indicating Job IDs.
+
+Protocols
+---------
+
+Each executor will have access to all plugins, directly, hence no need to do
+unsafe things like transporting code. The executor is NOT a generic 'worker'
+node. The transport between Jobs API <-> Dispatcher <-> Executor is zeromq.
+
+The transport between Executor and Storage is Redis.
+
+
+TO DO LIST
+----------
+
+In consideration to https://redis.io/topics/persistence, snapshots will be
+triggered after each important API call, and whenever the Dispatcher gets a
+"SNAPSHOT NOW" command over its zeromq control channel.
+
+The AOF (Append-Only-File) is a persistence method for
+Redis Key-Value noSQL storage system). It is comparable to a Changelog.
+
+The RDB (Redis Database) is the snapshot-style persistence method. This is
+what we will be using.
+
+https://www.fullstackpython.com/blog/install-redis-use-python-3-ubuntu-1604.html
+
+https://redis.io/commands/save
+
+https://redis.io/commands/bgsave
+
+PLUGINS
+-------
+
+* HtmlStats: uses numpy to create standard deviation statistical
+  calculations to detect excessive changes between different parameters such
+  as html tags, pagesize in bytes, etc.
+
+There are other plugins included.

api/.dockerignore

@@ -0,0 +1,6 @@
+*~
+*pyc
+DEADJOE
+tests/*
+.#*
+requirements.txt.old

api/Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3.5
+ADD . /code
+WORKDIR /code
+RUN pip install -r requirements.txt
+RUN chmod +x /code/ronald.py
+RUN ls -l /code
+CMD ["/code/ronald.py"]

api/requirements.txt

@@ -0,0 +1,4 @@
+redis==2.10.6
+Flask_RESTful==0.3.6
+Flask==0.12.2
+pyzmq==17.0.0b3

api/ronald.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Author: Arturo Busleiman <[email protected]>
+
+
+from flask import Flask, request
+from flask_restful import Api, Resource
+
+"""This is a comment.
+   Yes. Very nice.
+"""
+
+from ronald.api_assets import API_Assets
+from ronald.api_stats import API_Stats
+from ronald.api_jobs import API_Jobs
+from ronald.api_workers import API_Workers
+from ronald.api_notifications import API_Notifications
+from ronald.api_charting import API_Charting
+
+#
+# INITIALIZATION
+#
+ronald_app = Flask(__name__)
+api = Api(ronald_app)
+
+#
+# API Routing
+#
+
+api.add_resource(API_Assets,
+                 '/assets',
+                 '/assets/',
+                 '/assets/<assetId>',
+                 )
+
+api.add_resource(API_Stats,
+                 '/stats',
+                 '/stats/',
+                 '/stats/<assetId>',
+                 )
+
+api.add_resource(API_Jobs,
+                 '/jobs',
+                 '/jobs/',
+                 '/jobs/<jobId>',
+                 )
+
+api.add_resource(API_Workers,
+                 '/workers',
+                 '/workers/',
+                 )
+
+api.add_resource(API_Notifications,
+                 '/notifications',
+                 '/notifications/',
+                 '/notifications/<count>',
+                 '/notifications/<count>/<assetId>',
+                 )
+
+api.add_resource(API_Charting,
+                 '/charting',
+                 '/charting/',
+                 '/charting/<count>',
+                 '/charting/<count>/<assetId>',
+                 )
+
+if __name__ == '__main__':
+    ronald_app.run(host='0.0.0.0', debug=True)
+    # Remember to cleanup here [dbClose(), etc, etc]

api/ronald/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Author: Arturo Busleiman <[email protected]>

api/ronald/api_assets.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Author: Arturo Busleiman <[email protected]>
+
+import redis
+from ronald.errors import *
+from ronald.storage import Storage, Asset
+from flask_restful import Resource, request, abort as restAbort
+from pprint import pprint
+
+
+class API_Assets(Resource):
+    def __init__(self):
+        """
+        Class Initialization.
+        Assets API uses the Storage class, which is a redis abstraction
+        """
+        pass
+
+    def get(self, assetId=None):
+        """
+        GET method handler for API_Assets
+        """
+        if assetId is None:
+            assets = Storage.list_assets()
+            return(assets)
+        else:
+            return(Asset(assetId=assetId).get())
+
+    def delete(self, assetId=None):
+        """
+        DELETE method handler for API_Assets
+        """
+        if assetId is None:
+            abort(400, message=ERR_MSG_ASSETS_DELETE_NO_ASSETID)
+        return(Asset(assetId=assetId).delete())
+
+    def put(self, assetId=None):
+        """
+        PUT method handler for API_Assets
+        """
+        if assetId is not None:
+            restAbort(400, message=ERR_MSG_ASSETS_PUT_NO_ARGS_REQ)
+        # get parameters
+        js = request.get_json(force=True)
+        # validate
+        if 'assetUrl' not in js:
+            restAbort(400, message=ERR_MSG_ASSETS_PUT_INVALID_REQUEST)
+        # TODO: check if assetUrl is indeed a valid Url
+        # attempt to create new asset
+        newAsset = Asset(assetUrl=js['assetUrl']).create()
+        # Asset() always returns a dictionary. send to client.
+        return(newAsset)
+
+
+if __name__ == '__main__':
+    print("")

api/ronald/api_charting.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Author: Arturo Busleiman <[email protected]>
+
+from flask_restful import Resource, request
+from ronald.storage import Storage
+
+
+class API_Charting(Resource):
+    def __init__(self):
+        """
+        Class Initialization.
+        """
+        pass
+
+    def get(self, count=None, assetId=False):
+        if count == 'all':
+            self.count = -1  # Standard, use -1 to specify 'all'
+        elif count == 'latest':
+            self.count = 10  # TODO: define from config
+        else:
+            try:
+                self.count = int(count)
+            except Exception:
+                return({'status': 'ERROR',
+                        'detail': 'Invalid count parameter'})
+            if self.count < 1:
+                return({'status': 'ERROR',
+                        'detail': 'Count parameter must be > 0'})
+
+        print("self.count={} assetId={}".format(self.count, assetId))
+        if assetId is False:
+            pass
+            # return(Storage.get_chartdata(library='default',count=self.count))
+
+        if Storage.get_asset(assetId=assetId) is None:
+            return({'status': 'ERROR',
+                    'detail': 'Invalid or inexistent asset'})
+
+        # return(Storage.get_chartdata(library='default',count=self.count,
+        #                              assetId=))
+
+
+if __name__ == '__main__':
+    print("")

api/ronald/api_jobs.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Author: Arturo Busleiman <[email protected]>
+
+import zmq
+import time
+from ronald.storage import Storage
+from ronald.messaging import Messaging
+from flask_restful import Resource, request
+from pprint import pprint
+
+""" The API_Jobs class implements job management.
+    get() _might_ list upcoming jobs. (redis?)
+    delete() cancels a job
+    put() inserts a job
+    post(), if implemented, will allow to modify
+    an existing job's parameters
+"""
+
+
+class API_Jobs(Resource):
+    def __init__(self):
+        """
+        Class Initialization.
+        """
+        pass
+
+    def get(self, jobId=None):
+        """
+        GET method handler for API_Jobs.
+        Takes care of getting a list of assets that need
+        to be run at the current period, and sending that
+        list to the Dispatcher, via ZMQ (implemented in Messaging class)
+        """
+        workSpec = Storage.list_assets_without_current_stats()
+        # pprint(workSpec)
+        ret = Messaging.queue_jobs(workSpec=workSpec)
+        if workSpec['assets'] is None:
+            return({'status': 'ERROR',
+                    'detail': 'No available workers. Notify admin.'})
+        return({'status': 'OK',
+                'detail': 'Jobs have been pushed to workers',
+                'dispatched_work_specification': workSpec})
+
+    def post(self, jobId=None):
+        """
+        POST method handler for API_Jobs
+        """
+        pass
+
+    def delete(self, jobId=None):
+        """
+        DELETE method handler for API_Jobs
+        """
+        pass
+
+    def put(self, jobId=None):
+        """
+        PUT method handler for API_Jobs
+        """
+        pass
+
+
+if __name__ == '__main__':
+    print("")