From a26195597802ea2e1ecf4dd4fedc933d9f33b321 Mon Sep 17 00:00:00 2001 From: Pras Velagapudi Date: Sun, 27 Nov 2016 12:31:48 -0500 Subject: [PATCH 01/43] Added directory for ipython notebooks. --- README.md | 1 + notebooks/README.md | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 notebooks/README.md diff --git a/README.md b/README.md index 3c82e1c..cb2e2ad 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ $ pip install git+https://github.com/platypusllc/analytics.git * **API documentation** can be found on [ReadTheDocs](http://platypus-analytics.readthedocs.org/en/latest/). * **Usage examples** of this library can be found in the [examples](examples) directory. +* **IPython/Jupyter notebooks** using this library can be found in the [notebooks](notebooks) directory. [1]: http://docs.python-guide.org/en/latest/dev/virtualenvs/ [2]: https://www.continuum.io/documentation diff --git a/notebooks/README.md b/notebooks/README.md new file mode 100644 index 0000000..f15b7ef --- /dev/null +++ b/notebooks/README.md @@ -0,0 +1,3 @@ +# Platypus Analytics Notebooks + +This directory contains example notebooks that demonstrate various useful data analysis operations. From 2303643f389eb61f72e8d5f56ad43a6ff6766348 Mon Sep 17 00:00:00 2001 From: Pras Velagapudi Date: Sun, 27 Nov 2016 12:44:57 -0500 Subject: [PATCH 02/43] Added initial, non-working version of Data Interpolation. --- notebooks/Data_Interpolation.ipynb | 243 +++++++++++++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 notebooks/Data_Interpolation.ipynb diff --git a/notebooks/Data_Interpolation.ipynb b/notebooks/Data_Interpolation.ipynb new file mode 100644 index 0000000..4e0c88d --- /dev/null +++ b/notebooks/Data_Interpolation.ipynb @@ -0,0 +1,243 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "import numpy as np\n", + "import numpy.lib.recfunctions\n", + "import scipy\n", + "import scipy.interpolate\n", + "import pandas\n", + "import matplotlib.pyplot as plt\n", + "import platypus.io.logs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Import the data from the specified logfile\n", + "log_filename = '../logs/platypus_20161126_150201.txt'\n", + "data = platypus.io.logs.load(log_filename)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Available sensors/channels:\n", + " ES2, ec\n", + " ES2, temp\n" + ] + }, + { + "ename": "ValueError", + "evalue": "Unknown format code 's' for object of type 'long'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtypes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0;32mprint\u001b[0m \u001b[0;34m\" {:s}, {:s}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mValueError\u001b[0m: Unknown format code 's' for object of type 'long'" + ] + } + ], + "source": [ + "# Print the available sensors and channels for this logfile.\n", + "print \"Available sensors/channels:\"\n", + "for s in data.keys():\n", + " if s == 'pose' or s == 'BATTERY':\n", + " continue\n", + " for c in data[s].dtypes.keys():\n", + " print \" {:s}, {:s}\".format(s, c)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Select the sensor and the name of the channel for that sensor.\n", + "sensor_name = 'ATLAS_DO'\n", + "sensor_data = 'do'" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'sensor_name' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m# Extract the pose and the sensor data of interest.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mpose\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'pose'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0msensor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msensor_name\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;31m# Linearly interpolate the position of the sensor at every sample.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mNameError\u001b[0m: name 'sensor_name' is not defined" + ] + } + ], + "source": [ + "# Extract the pose and the sensor data of interest.\n", + "pose = data['pose']\n", + "sensor = data[sensor_name]\n", + "\n", + "# Linearly interpolate the position of the sensor at every sample.\n", + "sensor_pose = scipy.interpolate.interp1d(pose['time'], pose[['latitude', 'longitude']],\n", + " axis=0, bounds_error=False)\n", + "\n", + "# Add the position information back to the sensor data.\n", + "sensor = sensor.join(pandas.DataFrame(pose_interp(sensor['time']), sensor.index,\n", + " columns=('latitude', 'longitude')))\n", + "\n", + "# Remove columns that have NaN values (no pose information).\n", + "sensor = sensor[np.isfinite(sensor['time'])]" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABDEAAAI8CAYAAADoRDBpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmcZHV56P/PU9XdM8MwMGwyzDBsgiiLuCAoGh0TRWJu\nJLkxuMQrYBIlXhO9rkiSK9lw+WlUYjD+rkb9JQFRb2IgccPEdokLriyyo8MywLDDzDB0d1U9vz/O\nqZkzRS/VPb1UTX/er9d59alTZ6vqqlPnPOf5Pt/ITCRJkiRJknpdbaF3QJIkSZIkqRsGMSRJkiRJ\nUl8wiCFJkiRJkvqCQQxJkiRJktQXDGJIkiRJkqS+YBBDkiRJkiT1hYGF3gFJkiRJktSdiMj53mZm\nxnxvcyJmYkiSJEmSpL5gJoYkSZIkSX0kYv4SIzLnPfFjUmZiSJIkSZKkvmAQQ5IkSZIk9QWbk0iS\nJEmS1EdsTiJJkiRJktTjzMSQJEmSJKmPzGcmRq8xE0OSJEmSJPUFMzEkSZIkSeojtdr85SM0m815\n21Y3zMSQJEmSJEl9wUwMSZIkSZL6iDUxJEmSJEmSepyZGJIkSZIk9REzMSRJkiRJknqcQQxJkiRJ\nktQXbE4iSZIkSVIfsTmJJEmSJElSjzMTQ5IkSZKkPmImhiRJkiRJUo8zE0OSJEmSpD5iJoYkSZIk\nSVKPMxNDkiRJkqQ+Uqst3nyExfvKJUmSJElSXzETQ5IkSZKkPmJNDEmSJEmSpB5nJoYkSZIkSX3E\nTAxJkiRJkqQeZxBDkiRJkiT1BZuTSJIkSZLUR2xOIkmSJEmS1OPMxJAkSZIkqY+YiSFJkiRJktTj\nzMSQJEmSJKmPmIkhSZIkSZLU48zEkCRJkiSpj9RqizcfYfG+ckmSJEmS1FfMxJAkSZIkqY9YE0OS\nJEmSJGkKEbE0Ir4fET+NiGsi4t3l9L0j4rKIuCEivhoRKyvLvDMiboyI6yLi5J3ZvkEMSZIkSZLU\nlcx8FHh+Zj4FeDLw/Ih4DnA2cFlmPgH4j/IxEXEU8DLgKOAU4IKImHEswiCGJEmSJEl9JCLmbRhP\nZj5Sjg4BdeAB4CXAp8vpnwZ+oxw/FbgoM8cycz1wE3DCTF+7QQxJkiRJktS1iKhFxE+BjcDXM/Nn\nwP6ZubGcZSOwfzm+Gri9svjtwJqZbtvCnpIkSZIk9ZGFLuyZmS3gKRGxJ/CViHh+x/MZETnZKma6\nbYMYkiRJkiQJgNHRUUZHR7uaNzMfioh/B54ObIyIVZl5V0QcANxdzrYBWFtZ7MBy2ozYnESSJEmS\npD4ylzUwlixZwooVK7YN42x733bPIxGxDHgh8BPgEuD0crbTgS+U45cAL4+IoYg4FDgCuHymr91M\nDEmSJEmS1K0DgE+XPYzUgH/IzP+IiJ8An42I3wXWA6cBZOY1EfFZ4BqgAbw+M2fcnCR2YllJkiRJ\nkjSPIiJXrVo1b9u76667yMyFLcJRYXMSSZIkSZLUF2xOIkmSJElSH6nVFm8+wuJ95ZIkSZIkqa+Y\niSFJkiRJUh+J6JkSFfPOTAxJkiRJktQXDGJIkiRJkqS+YHMSSZIkSZL6iM1JJEmSJEmSepyZGJIk\nSZIk9REzMSRJkiRJknqcmRiSJEmSJPURMzEkSZIkSZJ6nJkYkiRJkiT1ETMxJEmSJEmSepyZGJIk\nSZIk9ZFabfHmIyzeVy5JkiRJkvqKQQxJkiRJktQXbE4iSZIkSVIfsbCnJEmSJElSjzMTQ5IkSZKk\nPmImhiRJkiRJUo8zE0OSJEmSpD5iJoYkSZIkSVKPMxNDkiRJkqQ+YiaGJEmSJElSjzMTQ5IkSZKk\nPlKrLd58hMX7yiVJkiRJUl8xE0OSJEmSpD5iTQxJkiRJkqQeZxBDkiRJkiT1BZuTSJIkSZLUR2xO\nIkmSJEmS1OPMxJAkSZIkqY/YxaokSZIkSVKPMxNDkiRJkqQ+Yk0MSZIkSZKkHmcmhiRJkiRJfcSa\nGJIkSZIkST3OTAxJkiRJkvqINTEkSZIkSZJ6nJkYkiRJkiT1ETMxJEmSJEmSepxBDEmSJEmS1Bds\nTiJJkiRJUh+xi1VJkiRJkqQeZyaGJEmSJEl9xMKekiRJkiRJPc5MDEmSJEmS+og1MSRJkiRJknqc\nmRiSJEmSJPURa2JIkiRJkiT1OIMYkiRJkiT1kVqtNm9Dp4hYGxFfj4ifRcTVEfFHHc+/JSJaEbF3\nZdo7I+LGiLguIk7emdducxJJkiRJktStMeB/ZeZPI2J34EcRcVlmXhsRa4EXAre0Z46Io4CXAUcB\na4CvRcQTMrM1k42biSFJkiRJkrqSmXdl5k/L8c3AtcDq8um/Bt7escipwEWZOZaZ64GbgBNmun0z\nMSRJkiRJ6iO9UtgzIg4Bngp8PyJOBW7PzCs79m818L3K49spMjJmxCCGJEmSJEkC4KGHHuLhhx+e\ncr6yKcnngTcCLeAciqYk22aZZPGc6f4ZxJAkSZIkqY/MZSbGypUrWbly5bbHt99++3jbHwT+L/CP\nmfmFiDgWOAS4oty3AylqZZwIbADWVhY/sJw2I9bEkCRJkiRJXYkiSvEJ4JrM/BBAZl6Vmftn5qGZ\neShFk5GnZeZG4BLg5RExFBGHAkcAl890+2ZiSJIkSZLUR8br+nQePRt4FXBlRPyknHZOZn6pMs+2\n5iKZeU1EfBa4BmgAr89Mm5NIkiRJkqS5lZnfZopWHZl5WMfj84DzZmP7BjEkSZIkSeojvdI7yUKw\nJoYkSZIkSeoLZmJIkiRJktRHFrgmxoJavK9ckiRJkiT1FTMxJEmSJEnqI9bEkCRJkiRJ6nEGMSRJ\nkiRJUl+wOYkkSZIkSX3E5iSSJEmSJEk9zkwMSZIkSZL6iF2sSpIkSZIk9TgzMSRJkiRJ6iPWxJAk\nSZIkSepxZmJIkiRJktRHrIkhSZIkSZLU48zEkCRJkiSpj1gTQ5IkSZIkqccZxJAkSZIkSX3B5iSS\nJEmSJPURm5NIkiRJkiT1ODMxJEmSJEnqI3axKkmSJEmS1OPMxJAkSZIkqY9YE0OSJEmSJKnHmYkh\nSZIkSVIfsSaGJEmSJElSjzMTQ5IkSZKkPmJNDEmSJEmSpB5nJoYkSZIkSX3EmhiSJEmSJEk9ziCG\nJEmSJEnqCzYnkSRJkiSpj1jYU5IkSZIkqceZiSFJkiRJUh8xE0OSJEmSJKnHmYkhSZIkSVIfMRND\nkiRJkiSpx5mJIUmSJElSHzETQ5IkSZIkqceZiSFJkiRJUh8xE0OSJEmSJKnHGcSQJEmSJEl9weYk\nkiRJkiT1EZuTSJIkSZIk9TgzMSRJkiRJ6iNmYkiSJEmSJPU4MzEkSZIkSeojZmJIkiRJkiT1ODMx\nJEmSJEnqI7Xa4s1HWLyvXJIkSZIk9RWDGJIkSZIk9ZGImLdhnG3/fURsjIirKtNOiIjLI+InEfGD\niHhG5bl3RsSNEXFdRJy8s6/dIIYkSZIkSerWJ4FTOqa9D/jTzHwq8L/Lx0TEUcDLgKPKZS6IiJ2K\nQ1gTQ5IkSZKkPrKQvZNk5rci4pCOyXcCe5bjK4EN5fipwEWZOQasj4ibgBOA7810+wYxJEmSJEnS\nzjgb+HZEvJ+ixcezyumr2TFgcTuwZmc2ZHMSSZIkSZK0Mz4B/FFmHgT8L+DvJ5k3d2ZDZmJIkiRJ\nktRH5rI5yV133cXGjRunu9gJmfmCcvzzwMfL8Q3A2sp8B7K9qcmMGMSQJEmSJEkArFq1ilWrVm17\nfOWVV3az2E0R8bzM/Abwy8AN5fRLgAsj4q8pmpEcAVy+M/tnEEOSJEmSpD6ykIU9I+Ii4HnAvhFx\nG0VvJK8F/jYilgBby8dk5jUR8VngGqABvD4zbU4iSZIkSZLmXma+YoKnTpxg/vOA82Zr+wYxJEmS\nJEnqIwuZibHQ7J1EkiRJkiT1BTMxJEmSJEnqI2ZiSJIkSZIk9TgzMSRJkiRJ6iNmYkiSJEmSJPU4\nMzEkSZIkSeojZmJIkiRJkiT1OIMYkiRJkiSpL9icRJIkSZKkPmJzEkmSJEmSpB5nJoYkSZIkSX3E\nTAxJkiRJkqQeZyaGJEmSJEl9xEwMSZIkSZKkHmcmhiRJkiRJfcRMDEmSJEmSpB5nJoYkSZIkSX3E\nTAxJkiRJkqQeZxBDkiRJkiT1BZuTSJIkSZLUR2xOIkmSJEmS1OPMxJAkSZIkqY+YiSFJkiRJktTj\nzMSQJEmSJKmPmIkhSZIkSZLU48zEkCRJkiSpj5iJIUmSJEmS1OPMxJAkSZIkqY+YiSFJkiRJktTj\nzMSQJEmSJKmPmIkhSZIkSZLU4wxiSJIkSZKkvmBzEkmSJEmS+ojNSSRJkiRJknqcmRiSJEmSJPUR\nMzEkSZIkSZJ6nJkYkiRJkiT1ETMxJEmSJEmSepyZGJIkSZIk9REzMSRJkiRJknqcmRiSJEmSJPUR\nMzEkSZIkSZJ6nEEMSZIkSZLUF2xOIkmSJElSH7E5iSRJkiRJUo8zE0OSJEmSpD5Sqy3efITF+8ol\nSZIkSVJfMYghSRIQEc+JiO9ExIMRcV9EfDsijl/o/ZquiNg9IjZHxBcr034WEZvKoRERWyuP3xkR\nZ0TEt6ZY7xkR0YqI08Z57pyI+Hm5vtsi4jNd7OdwuR8PR8RDEfHDiHhHRAx1zHdURFxS/l8ejoj/\njIhnTec9kSRpVxMR8zb0GoMYkqRFLyL2AP4N+DCwF7AG+DNgZCH3qy1KXc7+W8CtwLqI2B8gM4/O\nzBWZuQL4FvA/248z891drvd04Crg1R37djrwKuBXyvUfD3yti/VluR97AKuAtwAvB6rBl8cD/wVc\nARwCHAD8C/DViHhml/stSZJ2IQYxJEmCJwCZmRdn4dHMvCwzr2rPEBGviYhrIuL+iPhyRBxUea4V\nEa+LiBsi4oGI+EjlucMj4htlJsE91SyFiDgpIn5QPnd5NcOgzFT4y4j4L2ALcGiXr+V04OMUF/+v\nmmCead1WiYiDgWcDZwIvbAdHSscDX8nMXwBk5sbM/Hi3qy6X2ZqZ3wBeAjwrIn6tfP5c4L8y808z\n88HM3JKZfwP8A/De6bwGSZJ2JWZiSJK0uF0PNCPiUxFxSkTsVX0yIk4F3gn8JrAvRTbDRR3r+DWK\nC/onA6dFxMnl9L8AvpyZKykyPM4v17k38O/Ah4C9gb8G/r1j268Cfg/YHbg1Ii6NiLdP9CLKYMNz\ngc+Ww6snmDUnWscEXg18IzN/DPwQ+J3Kc98DXh0Rb42I4yOiPo317rAfmXlbuf7nlJNeCHxunOU+\nBzw7IpZMY1uSJGkWRMTfR8TGiKje7Pl/IuLaiLgiIv45IvasPPfOiLgxIq6rnB/NmEEMSdKil5mb\nKC6cE/g/wN0R8a8R8bhylrOAd2fm9ZnZAt4NPCUi1lZW857MfLi8EP868JRy+ihwSESsyczRzPxO\nOf3XgOsz858ys5WZnwGuo8hGoNyXT2XmteXzjcz89cx83yQv5X8Al2fm7cA/A0dFxFMmmb9br2Z7\nMOFzVIIjmflPwB8CLwKGgY2TBVq6cAdFUAdgH+DOcea5k+IcZu9xnpMkaZe3wJkYnwRO6Zj2VeDo\nzDwOuIHi5g8RcRTwMuCocpkLImKn4hAGMSRJAjLzusw8MzPXAscAqymyJAAOBj5cNhV5ALivnL6m\nsoq7KuOPACvK8bdTNJu4PCKujogzy+mrKWpXVN1STm+7bZovY1uwITPvowgqnD7NdewgIp5NUY/i\nn8tJnweOjYjj2vNk5oWZ+UJgT4qAz1/sxJ2WA4H7y/F72fH9aDsAaAEPzHAbkiRphjLzW3T8BpfN\ncFvlw+9T/J4DnApclJljmbkeuAk4YWe2bxBDkqQOmXk98GmKYAYUwYbXZuZelWF5Zn6vi3VtzMzX\nZuYa4HUUdyAeD2ygCI5UHVxO37Z4t/scEScBhwN/EhF3RsSdwLOAV06ziUen0ymCMFeV6/xBZfoO\nMrOZmZ8HrgSOnu6GysyWp1E014GiQOhvjzPracB3MvPR6W5DkqRdQY/XxHgN2wt1rwZurzx3Ozve\nBJq2gZ1ZWJKkXUFEHEnRvOPizNxQXky/AvhuOcvfUWQXXJGZ15TtPE/OzPHqNUClcGZE/Dbw3bKJ\nx4MUgYkm8CXgbyLiFRTZE78FPJGil5THrKcLp1OkclbrYOxGEVD41S7WG2WNic7nTgN+n6J+R9tL\ngf8dEW+jaMJyN0XgYQtFs5KjKe7CTCXKDe8GPAP4IPD9zGyf+PwZ8IOI+EvgA0ADOKPc5gu7WL8k\nSZqmW265hVtv7UwW7U5E/DEwmpkXTjLbdGtz7cAghiRJsAk4EXhzRKykCDZcCrwNIDO/EBG7A58p\ni2c+RBEwaAcxOn+MszLteOCDZeBjI/BHZTolEfHfKLp1/ShwI/DfMvP+jvVsExFfBL6Zme/pmL6U\nImPhf2Tm3R3P/QNFYKMaxBhvf08CtnZMezVFYOL/y8xmZZ2fBP6cImDxEHAORVvXOrAeOKtS+2My\nH4mID5bjN1G8nx/YtgOZN0XEc4D3lOutUWSCnJyZ30WSJM26gw8+mIMP3p4s+u1vf7ur5SLiDODF\nwK9UJm8AqjXEDmTHrNNpi8ydCoJIkiRJkqR5EhF5zjnnzNv2zjvvPDJzh0zNiDgEuDQzjy0fn0Jx\nI+J5mXlvZb6jgAsp6mCsoWgqenjuRCDCTAxJkiRJktSViLgIeB6wb0TcBryLojeSIeCyso7GdzPz\n9WUz3M8C11A0C339zgQwwCCGJEmaIxGxmfHbvZ6Smf813/sjSdKuYoYFN2dFZr5inMl/P8n85wHn\nzdb2DWJIkqQ5kZm7L/Q+SJKkXYtBDEmSJEmS+shCZmIstNpC74AkSZIkSVI3zMSQJEmSJKmPmIkh\nSZIkSZLU4wxijCMi3hIRrYjYe4LnT4mI6yLixoh4R2X6X0TEFRHx04j4j4hY27HcQRGxOSLeUpn2\n5XL+n0XEJyJisJx+VkRcGRE/iYjvRsRxXez330fExoi4auavXpIkSZLUyyJi3oZes2iDGBGxLiI+\nOc70tcALgVsmWK4OfAQ4BTgKeEVEPKl8+n2ZeVxmPgX4AkV/uVV/Dfx7x7SXZuZTMvNoYE/gZeX0\nf8rMJ2fmUym6o/lAFy/rk+V+SZIkSZK0y1m0QQzG77ceikDD2ydZ7gTgpsxcn5ljwGeAUwEyc1Nl\nvt2Be9sPIuI3gJ8D1+ywE5mby+cHgaH2MlOs620RcXmZ9XFuZV3fAh6YZN8lSZIkSepbUxb2jIiV\nwLOAQygu/NcD383Mh+Z0z+beY/JiIuJU4PbMvHKStJk1wG2Vx7cDJ1bW8VfA/wAeAZ5ZTtudIjDy\nAuBt42z3K8AzgMsy88uV6a8H3gwsB04qp50MHJ6ZJ0REDfjXiPilMoAhSZIkSdrF9WIzj/kyYSZG\nRPxSRFwCfBN4OXAQRSDjFcC3IuKSiHjOvOzlLIqI70XET4D/A7ykrDnxk4h4CfBOdmwCMt4nY6IM\njuLJzD/OzIOATwEfLCefC3wwMx8Zb52Z+SLgAGBJRJxemX5BZh5OEcj4+3LyycDJ5Wv4EXAkcPjk\nr1qSJEmSpP43WSbGbwJvycwbx3syIp4AnAV8ey52bK5kZjs74nnAGZl5Zvn4GOBQ4IoyqnUg8KOI\nOCEz766sYgNQLdi5liIbo9OFwBfL8ROA34qI9wErgVZEbM3MCyr7NRIR/5ciq+PTHeu6GPi7yuN3\nZ+b/O42XLUmSJEnaRdRqi7cyxISvPDPfPFEAo3z+hsx882Qrn6gXj455zi+fvyIinjrVshGxd0Rc\nFhE3RMRXy+YuRMQhEbG1kllxwXjbq2664/VcnZn7Z+ahmXkoRWDiaR0BDIAfAkeU2xuiKMR5SbkP\nR1TmOxX4Sbnu51bW+yHgrzLzgohYHhEHlMsOAP+tvUzHun4NuLIc/wrwmohYXs63JiL2m+K1SpIk\nSZLU96YM30TEmyJizyh8ogwQvKiL5SbrxaM9z4sp6jscAbwW+GgXy55NUTviCcB/lI/bbsrMp5bD\n66fYxWTypiHbnouI1RHx7wCZ2QDeQBFMuAa4ODOvLWd9d0RcFRE/BdYBb2Fyu1PUtLgC+DFwK9ub\njfzPiLi6bDbyh8CZ5fYvo8jy+G5EXAl8rlwPEXER8B3gCRFxW0ScOcX2JUmSJEl9ZjF3sTplYU/g\nNZn5oTJwsTdF0cp/oLiIn8y2XjwAIqLdi8e1lXleQtl0IjO/HxErI2IVRbOOiZZ9CfC8cvlPA8Ps\nGMjoSmZ+A/jGJM8fVhm/gyIbov34S8CXxlnmpV1s988q4xsp3qfx5nvTJOs4Hzh/nOmvmGr7kiRJ\nkiT1q24a0rRDL78G/ENmXt3lusfrxWNNl/OsnmTZ/cuLf4CNwP6V+Q4tM0WG+7HoqCRJkiRJUzET\nY3I/ioivAocBZ0fEHkCri+Um7cWjopt3JcZbX2ZmRLSn3wGszcwHIuJpwBci4ujM3NTlfkiSJEmS\npB7WTRDjd4HjgJ9n5iMRsQ9lfYYpdNOLR+c8B5bzDI4zfUM5vjEiVmXmXWVRzLsBMnMUGC3HfxwR\nNwNHUNSa2KYS9JAkSZIk7YIys/dSCGZRL2ZIzJdughgJHE3Rc8afA8uBpV0st60XD4osiZcBnTUb\nLqEokvmZiHgm8GBmboyI+yZZ9hLgdOC95d8vAETEvsADmdmMiMMoAhg/H/cF7eIfaM2viDg3M89d\n6P3oZTE8HMBewCpgH4quhleW0/YBHlcZ9idzf4qsrynt02rxic2bu0rp6hcXXXQRr3jF9ErcLOYf\nMk3uwgsv5Hd+53cWeje0C7nwwgt55StfudC7sUvI9N4aTPyZ6ub9mez37+p6nXN2223qdWRmNho/\nZ3DwRmCkHEbLv1srw73ATcDNwPpct250ypVrQXjjetfWTRDjAqAJ/ApFEGNzOe34yRbKzEZEtHvx\nqAOfyMxrI+J15fMfy8wvRsSLI+ImYAvbe+AYd9ly1e8BPhsRvwusB04rpz8X+POIGKNo7vK6zHyw\nmzdB0szE8HCNIjBxQDlUx7cPrdZqarVujjfliru/IL+vVuNTS5bw3LExDm61ujqo7YoW+kTYIEpv\nW+jPx3xYjJ/Bhfq/Zuai+Ez1u377H7X3t/pd3tnv9eUD3Z0VZEQwOPh44PHTWH0rhodvZXtQo/33\nZooM8vtz3br++ieoryzG3722br7ZJ2bmU8uuPsnM+yNisJuVj9eLR2Z+rOPxG7pdtr194AXjTP9n\n4J+72S9J3SuzKNYAxwDHDN5zz3MOWr78BVuXLl0+EEGjmwNorZsawjuqZ7Iykz3Lv3u0Wtse753J\nRUuWcFe53i8sWcIXlixhaSaHNJvs32qxTybLMlkKLM1kaSZLKKKi215bx/bqFAfFeiYDwLJMds9k\nt47l9FgLebK8mH/EtV2/XbCpd/jZmRvTPTbPxbH88GZz2/gerRYf37KFByJYX6/zi1qN9bUav6jX\n2TiD8xSKDhIOKYfHXJsAozE8fBdw5zjDeuA6YIOBDmn6uglijEbEtvP3iNiP7gp7SovF8ELvwEys\nXLnycaeffvp7XvCCF5wJRbrVPRHcUatxR63GLbUat9Tr7Far8UjlxGJsv/24eQbbW1YGH/ZotVgO\nLM9keRkkWJnJnq0We1aCFrtnTtp9Ui2TD3SkiD4awXUDA1w3g/3rZv93z2SPch/3KPd32+Pyb/t1\nLKO7qsXjOeaYY2Zz13d5C3UB0i/Bk2OPPXahd0F9rvM7dswxx3jhvwjM5zFuro5TJzYaLM9kSwQP\n12p8Z2CAX240OKDR4FmV+bYCt9dqPFjenBkDGsBoBKOVv/dHcGetxp21GvdFkJO/R0PAQeUwkXti\nePgLwOeA/8h167zGkroQU/0IRcSrKJpsPB34NPBS4E8y87Nzv3uzLyLSmhja1QwNDS359V//9T86\n44wz3jfe89Xv+QjFD/VttRq31evcVqtxe63Gxlqtu6yKDisy2bvVYu9M9spkr+p45XE3hXTG29/J\n3FKrccXAANfX61xfr3PPzO6kzImBMkgzCAxlMsT2bpaSIhLc/rsEiuBNq8VeZRCnOl6jCDI9GsGm\nyvBwx9+EHbY31LH99t9xp43zOICRiKJxcMSO4xTZKUvLgM3uZZBnReV1d0qKE8XNEduGhyuv59EI\nmuVrbVbGG0Argkb5uAU0yufrwG6Z7FZue58yC2ffVov9yuBS73wqJGn+9Eugda59ZmiIC5csAWBN\nq8UFW7bMSh2tUWBjrcZd5Q2gu8rgxsYI7q/V2Dr99/9y4Ddz3bo7ZmH3Fr1d/ZovIvK8886bt+2d\nc845PVVXcspMjMz8x4j4EUVNDIBTK/UpJM2B5z73uS8/66yzPrJixYp9xnu+8yL/85///GOmPwLc\nVq9vD1iUQYu7p75z8BjLMzm42eSgVos1rRarWi32z+RxrRZTl8uavuqJ12QBjYNbLQ4e3V5T657y\nDsndtRoPlBfFj1Jc/LfH22urvgcJO1wwN8u7MFsj2FIO09WI4KFFfAI5WDbPqVM0z0lgSwSteX5P\nBsuAxj7tpkHtgeJzvVtlWA47PF7KzLNppmMMuKlMZ74vggci2Fp+BsciWFoGhlZk7hCg2bfVYsk8\n7J+khWMgYuf9+ugo/zI0xNYINtRqXFer8aTWzic8DAFrW62iO8VKs5W2RykyNx4oz0nuL4/v95fB\njttqtc7zixOe2GxuOOmFL3zVdy677J92egelXdiUmRgAEfFLwOGZ+cmyOcnumfmLOd+7ObCrR+XU\nW9auXfuks84662+f/OQnP7+b+WeSnrsJtmVUVLMr7ptBVsLeZYBidavF2larCBKUmRSdX5rqvs71\nSdZCpy032Z5B8FCtxsNlgKL9d9t45bkRTzz7Xq0MduxW1lQZZHvGymAZ9Nij1WJFmfEx3t/xggyb\ngZvrdX42MMDPyiyisRl+XvZotdg3k/1aLfZttViVyQHtQGOrxdCMX72knWUAonf8zZIlXDZUHBFP\nGR3l9SMjC7xHRVbhdfU63xoY4IuDg9turvzG6CivKffv3nvvve3MM8+crDmKJrCrX/NFRL773e+e\nt+29851AV77NAAAgAElEQVTv7K9MjIg4l6IpyZHAJykCj/8IPHtO90zqMXvuued+r371q887+eST\nf6/bZS644IJZq+L+KHBLvc4ttRrr63VuLYMWD04zWFHL5IBM1jabxR2EcljdarGsy3V0vp7xKorP\npohY0EBGHdidosnEqnHutoynnQFSbUsLxZ39oKgG1h7fCjxYtsXdNlQeJ8XBepCi+U77wrk67FFm\nPrS3NVYdL5t/jHXsT+c8ox2Pk6J5yVJgSXlBviRz23iTIltlawRbgE2VZiITZVy0swrazU+qF/1L\nc8fiqrX2OMXndtt4ZZ5GBI9QZHk8HMF9tRr3R3BvrcY9tRqbd+Iz2YpgM+zUOoYqr3FrGeCaQYrx\nhB6u1XgY+Hn9saVnI5N9y6DG6lZr23f+wAkCk5Iey0DEruH5jca2IMaPBgbIkZEFPwbWgKOaTY5q\nNtk3k0+XTV6+MDTEUc0mxzUaXLlq1dozvvKVfEajwUGtFueee+6v/uhHP/rywu65tPC6qYlxBfBU\n4EeZ+dRy2pWZ+eR52L9Zt6tH5dS9l73sZX/8yle+8s9qtdpOdTwx2xfXTeDOWo1by2F9vc76sq3l\ndJqBDGRuy6hY22pxUCVY0VX3QhPo4pixE2vfuW2rN7QomuW0a1g0IwiKjIad+ezNxCPAvWVgY0sE\nj1SGLbDD4/a0djOi0Xm8eFndbHJoq8XeZdOXdl2RgUwerdQPubdW497y730RNGe4j8szObASyNyn\nIzC2vAxe2SuP+p1BCEHxe/Q7u+++LYj8d5s3s7qHzilawF8uW8YPJ+gSdiCTv9i6laMrN1JuueWW\nq9/whjdYuXkCu/o1X0Tke97znnnb3tlnn91fmRjASGa22j8CEbG825VHxCnAhyjOgz6eme8dZ57z\ngV+lONc8IzN/MtmyEbE3cDFwMEX3RKdl5oOV9R0EXAO8KzM/0O2+qn8dd9xxz/+DP/iDj65Zs+bI\n6Sz3qle9asLnkuIHb4TijvooRY2DFtsLCzYrj0fKegtbO2owtIs3dq67c3wkgnsjuLvsGWQ6qeVD\nmaypBCnWlndcV5V3rudbZu6yGRnqTo0iZW9bU4YF/J/tBhzUak1aGn4iDcogB8V3tDNjZXOMX2C1\nOj5esdyhMsD4pGaToxsNjmo22WcG71ETeLCddVIeP+6qFJe7Z5L6N1siuH5ggOun2MZQbu+qeFvv\nPJUmNO2eedrjK8veh3rmLEd9zyCEZsMAcEyzyQ/KIMEVAwOsHhtb2J2qqAFv2rqVV61YMe7zjQg+\nvWQJ73vkkW3TDj744GMuvfTSbT8eb3vb25513XXXfW/Od1bqAd1kYrwNOBw4GXg38Brgwsw8f4rl\n6sD1FP0mbwB+ALyiWhQ0Il4MvCEzXxwRJwIfzsxnTrZsRLwPuDcz3xcR7wD2ysyzK+v8PMW53eXj\nBTF29ahcv1u7du0TX/va157/lKc85YVzsf4xipP+dmGlB9rjlWn3lxcgM73DOVdqZbDikLJOxSFl\noc39yhT8+TCdAIIZGVrs2r2xtAMbS2FeL/LHKCrn31n2QNTuiei2en2HbpNn20AZ0Gh3l7xneZxa\n02pxQKvFmmaT3eds6+o1BiHUKy4ZHOTjS4u+0k4aG+PsRx+d0XqabG8SOltur9W4eGiIbwxOnq/4\nyc2bpwx6b926ddP73//+V15++eX/Nou72Hd29Wu+iMj3vvcx+QFz5h3veEf/ZGJE8ctzMfBEivqB\nTwD+NDMv62LdJwA3Zeb6cl2fAU4Fqj2bvISi21Yy8/sRsTIiVgGHTrLsS4Dnlct/GhgGzi7n+w3g\n58CWLvZP44jh4aD4XCwph6UUiQePUpyPj+S6dV1dQS5btmz5WWed9be//Mu/fPpE8zQp2+VT3Nn8\nk49+lNEIrqPIZmhnNoyW7fM7B9jeTWWL7anhm8u08Hb6+JaYeS8TC2HvSlHNg5tNDikzLBaySN90\nAwdmZGixC9hWGHT/Bfi8DgIHljUwnlGZnhQV89uFgDeU9VfamSWbymPvVph2T0ZQ3DG8L4L7Jpln\nj3ZAo+Pvqjnq8Ug7x0CEdgXHVppiXD9OHaGpjAB/u3Qp3xwYoAbsk8nTGg2e1WhwTLM5o+zXW2s1\nPjs0xLcGBro63l5dr/O8RmPSeZYtW7biT//0Ty+99NJLAfjwhz985vDw8D82GlMsKPWRbr5vX8zM\nY4CvTnPda4DbKo9vB07sYp41wOpJlt0/MzeW4xuB/QEiYnfg7RTZG2+b5r7usmJ4eC9gFbBfOTxu\nkvE9KAIXkx5Fh77+dYaAZWWa8TKKk/TqhymAJ116KV8DvthZXJDtxQN7LduhaqAsXjhU/h0oCw3W\n2F5csFZmQSyhKFjYTrteWilSWBUTjA8C+7RaPC6TVa0W4ycT9h8DGVLvCYqT732aTZ4ySaHapGxS\nVwaD2wGOdm887WFTFAVoHyqDIY928Z1/uFbj4Vpt3OYsK8uARnvYv+zSef9Wi5UWJJ02AxBS4aBW\ni6VlnaH7yrpC02nK96XBQYbLTIkWsDGCLw0N8aWhIVZk8oxGg2MbDZ7QarFXWSy98zywCWyo1fhh\nvc73Bge5fpx6ZyeMjXFfrcbN4wRafjowMGUQo9Mb3/jGT77xjW/85KWXXsodd9xx4+te97onTGsF\n6lmL+fg+aRAjMzMifhQRJ2Tm5dNcd7dHhW7e/RhvfeX+taefC3wwMx+JKf6jZY8rbcOZOdzdrvam\nGB7eg6L3mCPGGfaa7e2NRTAGfZPVUFUrU5z3ymSvVou9Mtm7Y3zv8kR5vgsQ9vKBaGeCBXMZyJA0\ndwK2BWVXTuMYMELRbO+h2N7Lzp1ls5Y7yr+TFU19sOx16dpxnhvK5HFlYONxZc8rB7ZaHNhssl8Z\naN7VeTyVZqYOHN5scnVZF+P6ep2TphEQuGWS7I1NEfzn4CD/2dEcZLByU6t9M2+8WkkAT280eNnI\nCE9stdgM/K/ly9nY0QPdj+t1WjDjY93q1auPqNbROOecc9ZdddVV35jh6npKRKwD1i3wbmiedJOJ\n8UzgVRFxC9ubaWQXvZNsANZWHq+lyKiYbJ4Dy3kGx5m+oRzfGBGrMvOuiDgAuLucfgLwW2XNjJVA\nKyK2ZuYFnTuWmedOse89qWzqsRZ4SjkcR6NxPAMDs9p/dK28gB+iOPgmO3a7OJuisq2hzGKbbM9q\nWFaOD7H9gN3uljLKO3LVYbdK9427UVTgX16Z1o6K99Id/MVwQmogQ1o8lkCROZEJrdZjnm8B90Vw\nRxnUuKMS3NhYq014gg/Fb9Ht9Tq3j3MxsaSsu9EOahxYNsM7YCd7ZFoIHi+lufGEShDjhlqNk6ax\n7MrK8ew3Rkc5odHguwMDfHdggHsn6O5+LGLSc+daJsc3m7xsZIQjKuvfHXj71q28Y7fddjgmPlBm\ncZzQZXfvUznvvPOG281Obrzxxh+8+c1vPmFWVrwAypvSw+3HEfGuBdsZzblughgvmuG6fwgcERGH\nAHcALwNe0THPJcAbgM9ExDOBBzNzY0TcN8mylwCnA+8t/34BIDOf215p+aHdNF4Ao9e94x3vuPjZ\nz372SyOilsBttRpX1+v8rF5n73qd+zsPkhN0xdQ2lMk+7SJrrda2SvJ7jjO0u/SbKM4cZW8b7RoW\nWyvtph+JoMmOdSrazSwGyyBEO0ixJGJbzwUDdF8YKTMnDD70UlBiOvrhRHW23tu5CGTYpETqPzVg\nv0z2azY5ruNEvAncW8ncuKsMbLSHyTIARyL4eb3Oz+t1qNwNrZXN9Nr1QdZWghy9Wn9jZ45r/fC7\nIi2UJ1QCBTdMsy7GQZVlr6vXec3ICMc0m/zeyAg31Wr8aGCAG+p11tdqPFLWdGuN833cq9XiyGaT\nExsNntFssscE3/cjWi1+d2SEj5XFSNs+s2QJT3vkkVnvge6II454RjtLY3R0dOtb3/rWE3/xi19c\nNcub0SyqTRA8Wwy6+fw/PM60TVMtlJmNiHgD8BWK69lPlL2LvK58/mOZ+cWIeHFE3ESR5XHmZMuW\nq34P8NmI+F3KLla7eA1dKTMd9gD2BTbnunUbp1hkUqeddto5p5122h8vWbKkq/OkJnDAc5/LpfU6\nV9frXFOv83AXH856per76nJotyXee4L02m5Pcjrnq7G94ueKzB26ToyIyde7kydW1fW3xrm7p9ln\ngEDSfKpTZnFMUK9jM3B3JahxR9nryoZajYcm+L1sRXBHvc4d9Tqd7WL3LgMb7d/ONWXvKY+bx16f\nZtt8HbcNlqgfHVk5rtxUr9Nk4pt3nZ7ebFLPpBnBdfU6d0WwqswKPqLV4ojR0R3mT4qbfiMRNChu\n5g1SnEN368VjY1xTr/OtSmD2pnqd/75iBX+3eTOr5+j7PjQ0tOz888+/8tJLL2VkZOSRz33uc+dd\nfPHFfzUnG5NmoJsuVtcDBwEPlJP2Au4qh9/PzB/N5Q7Otmp3OzE8vIQio+OXgQMoXlv1WPa3uW7d\nGwCe/vSnv2jlypWr3vSmN31qsvVP5+RhDPh5mWXxs3qdawcGpqwzsVsmhzWbHNpscmirxaHNJmt3\nIlV2spOQqU5QphP925mTnYne02ogYyY9Zyy0fjgBnIv3aS5edy/8PyUtrIcjuL3SneyGsveVu2dw\np2qgLCS6phrcKHuLsnvY6emH3zotLmcuX8595XHh/C1bOGQaN8b+fNkyflhmQb9qZITTOgIXc2Er\n8PvLl497Y/NfNm2a94Dru971rlN+/OMff2WeNztti6GL1Q984APztr23vOUtPdXFajdBjP8DfD4z\nv1I+Phl4KfBJ4MOZ2VdtpyIi+frX9wLeSdGbyaQ+vmkT+03jAmmi97MB3FmrcUu9zg31OtfX69xc\nr0/a9hdgRavF0c0mRzcaHN1ocHCrNaOD1XSDFRPNP1XgYi5PVqrvbWcmRr8FMfrlpG6u3qfZfv0L\n/f+U1LtGKHoDuK1WK+pplIGOO6aovzGRfVstDmk2Objyd3Uf1t3oFf3ye6hdx18sW8YPykDE27du\n5TnTKO759YEBPrhsGQCrWy0u2LJlzgoK3xfBDwcG+GG9zvcHxz/CvHxkhFfOQyBlIuvXr7/yD//w\nD49bsB2YhEGM2dVrQYxumpM8KzN/v/0gM78aER/IzNdGxNAc7ttcemDqWQq/t+KxnV0+ZWyME8qg\nwn7NJkMU9SE2RbClVuOBCO6u1binVuPuWo076nU2dHmytFertS1gcVSjwYGt1mMOjhNdrk12IpAd\nzT46n5vqJGKypiI7k80xkelclM7kAnahain008naXL4/s10fw9oYkiayBDis1eKwVgsqFytN4K5K\nYdENlb+PqT9VcW+txr21Gj+sTGs361xbZmsc1GpxUKvFqhneeFhMujl299Nvp3rfAZUbYXdNM1Pr\nWY0GH8vkkbI48VX1+mNq++yMByL49sAA3xoc5LouanZ8ZskSrqjXeeOjj85Z05LJHHLIIU+u9nby\n1re+9ZnXX3/99+d9RxapxXxs7CaIcWdEvAP4DEUNxtMoegipUxQZX3R+OjjITyeIiE7XqmaTIxsN\njhob46hGgwNarccUuqy+yZMFE8Y7ERhv3uleQE60zelkcExXez3jvaadaUaykBbqQDOTgMF8vK/2\nWCJpIdVhWzORTlspsifbGRsb6nVuK7M5xrsh0Yzg1nqdW+t1/qtyfjCUyYGtFoc1mzy+2eTxZfbG\ndNrEa+rfJH9LNB2rKt/5u6f52VkKPH9sjH8fKu7jfnlwcKeDGJuA7wwO8q2BAa6u18ctBgrw+GaT\nm8cJbFw7MMBZu+/Onz3yCE+dxYDKTLz//e//Xru3k5/97GffPPvss5+3oDukXVY3QYxXAu+i7AUE\n+C+KnkLqzGJRzcXk2NFRnj06yjNGR9lznB/m8X6qqxf1U/2YV5t8TJSB0XkBOZ0Lys75ZjMTY6rX\n1k9Bi6qFOMGqvlft8W72Yz7f4+ns11TMxpA0W5ZRyd6oaAB31GqsL5uH3lKrsb5e554J7uaOVnpM\n+Vo5rZbJ2laLxzebHF4GNg41sLFTZpqRqcVpeeXzsnUGn4NTKkGM7w0M8EAEe03zM/gIcPnAAN8c\nHOQn9TrNcfajnskxzSbHNxoc32iwJpMG8N/HyRIHeNduu/HPmzbNeq8lM3X00Uc/t52l8alPfert\n//Zv//aR0a98ZT/gJOAE4J9y3bq+qq3YaxbzcWzKz3lm3gO8ISKWZ+aWjqdvmpvdmnPvBPak6K71\ne7luXQLE8HCdomvmFcA+wDEU71ETWAO8EPiVnd34VUNDXDU0xCfKLuYe12rxuGaT/cu/a5pNVjeb\nXaegdn6Aq5kK4wU0qgGRbj781Xm6CWDszBeqeiE63kV4lResE+umK9qJsnQkSeMbgG1NRapNUx4B\nbqvXubVWK4ZyfLxmKa2IIgBSr/Of5bRambHRztY4vNnkkGaTZfPyqhan6f7eLeaLhV1NNWA4MoPl\nD261OKrR4JqBAZoRXDY42FWBz3siuHxggMsHBrhqgrp4UQYufqnR4KSxMfboeH4AeP2jj3JBR7er\nbefsthtv2rp1QZqWTOaMM8543xlnnPG+j4yO8tWhbdUINgEGMTQjUwYxIuIk4OMUF/ZrI+I44HWZ\n+fq53rm5kuvWvWeC6U3goXK4HbiiY5b3di4Tw8O7A08EnsTIyJP33rTp1N323POIe+p1RqZoZzcW\nwR0DA9wxznNDmRzSaHBoo8FhY2Mc1mxyYLP5mH/YRHefO7sinaggZzWQMVVQY6oAxkQBjcnW2U1X\nqdWgxngBjl7Xqyc+vfQe2rREUj/bjaLrxiM7Urk3A78oMzFuqte5uWyekp03HyrNUb5eTouyzkY7\nW+PxZc9kXfXZrlk32W+mv1/9ZaDyv5xJcV+AF42NcU1ZHPSrg4P81ujoY24+NoEbazV+XAYufj5J\njYsjm01+aWyMZzca7DPF+dmLxsYmDGJcV6/zxuXLOWNkhF8dG5uzoqMztWLH12ZTk520mI893WQc\nfQg4BfhXgMy8IiK6+tBFxCnl8nXg45n52CBAxPnAr1LcyDgjM38y2bIRsTdwMXAwsB44LTMfjIgT\ngI+Vq60Df5WZF3eznzsj163bDPywHADeBhDDwwHsXe7nIcDBg3fffeKhS5f+aqxcuceGZpPNk3zw\nRiO4YXCQGwYHoayCPJjJQY0GBzcaHNhssrbRYG2rxd6VOhqdtSTGC2ZM54JxouDHZNkZky033nzd\nBjIMYHSnn96fqtkIZNikRFIv2R04ttnk2Epw4xGKwEY7qHFzWfy7M7CREUVvKvU6w+W0yGR1q7Ut\nW+PIstaGPaMsrM7fncV8YdEPZuO/8+xGg49nsqks5v/Tep1jmk1uLYt9XjUwwDX1+qTNVQ5rNnl2\no8EvjY2xahrnLgH870ce4c93Gz+kORLBx5Yu5YZ6nTc9+uisvN7ZctuO1wedGf5S17pqNpWZt3Yc\nkKfsi6gs/PkR4AXABuAHEXFJZl5bmefFwOGZeUREnAh8FHjmFMueDVyWme8rC46eXQ5XAU/PzFZE\nrAKujojPZ+aCVLgpm6jcVw4/Hm+eGB7ekyLAcShw2P633fbyA/bf//gbm83YsuyxSaRjEdw8OMjN\nHUVFl2SyqtlkVbPJAdWh1WJlJsFjm5BMJwPjMfs9SQCj2+BF5zITdZnaeWLQTxeonsRMnxkZknZ1\nu0HRdXolsLGVIrBxc73OTZXARmeBv4xgQ73Ohnqdb5bnAkOZHNlsclSzyVGNBkfaDGXBGdToHzM5\nq9wE3FqvU73A+LPddiMyHxOMrBrI5NhmkxMbDZ7RaLDfTpzTHt9sctLYGN+ZpKOBrw8O8rRGg+dN\nowvZudYRdvnpwuzFrmMxH1u6CWLcGhHPBii7VP0j4NrJFwGKgi03Zeb6ctnPAKd2LPsS4NMAmfn9\niFhZBiAOnWTZl7A9/ejTwDBwdmZurax3GfDQQgUwupXr1j1E0WSl3Wzlr9vPxfDwKuCpwNOAp9No\nnMDAwJrx1jMSwS0DA9wy8Nh/57Kyi7eDGg2OaDY5otHgkFaLoY5ARrcmCmCM12xkqnVPp87FbAcv\n5joYspgPKjvLQIakxWYZFEGISmDjUcrARhnUuLnsIaUzsDEawVUDA1w1MABLlmyrr3FQ2d3rwa0W\nBzWb7J/Zc6nli8VsFrHWzqt+Dya7UEjgjghuKL9/t5Q9FE3UBfN4AYx9Wy2eXBbnfFqjMavNwV47\nMsJPBgYmzfb40uBgTwUxHt9sMrw98HLgQu6L+ls3QYw/AD5MUdhyA/BV4H92sdwa4LbK49uBE7uY\nZw2wepJl98/MjeX4RmD/9kxlk5JPUgRBXtHFPvasXLfuLuBL5QBADA/vQxHYOLocjiqHvSZaz9Za\njV/UavxiYIBvlNMGMjm02eSIsn3tQa0Wa5tNls+gHkZn8GImP9BTZWF009xEhX7KVJnMzgQybFIi\naVewFHhSs8mTmk0YGwOKIoTtwMYN9TrXDgywseOCqlpf49uVu7RLyuDGmvI3f02rxdpWiwNaLZuj\nzBOD9L2h2jPgLbUajwJDwF1lb0K/qNW4qV7nxnp90qbf41nVanFk2YTs2EaDVWVG9FzYO5OzHn2U\nD46Tvd12zcAA71u6lFePjLBXJjfX61w0NMQV5Y3PXx8d5dUjI/PWO9JNO9YFuXOeNqtdULe9k7xy\nBuvu9iqim+92jLe+zMyIyMrjy4GjI+KJwJcjYjgzH3rMyiLOrTwczszhLvd1QeW6dfcBXyuHbcrg\nxhETDJ2FjWlEcOPAADd2ZG7sV2Zt7J7J7sDumewN7JPJfsC+FAfMoc4VzuS1TJGFMV4AY7IL08mK\nm843T1Bmhyd7krSjJcATm02e2Gzya2Vg494IrqnXuaZsg3/LBMUDRyK2ZXRQCW7UMlnVanFgx7DW\nIqKzzt+03nBQea67OYIHajVOW7GCWuZjspwmMljJdmp3twywR6vFR7ds6bp3wZ2VFN/rqXx7cHCH\ngGbVpUNDbKzV+JOtW8d9fjbdXKvxzR2vPT4/m+uPiHXAutlcZ69bzMeUCYMYEfE3lYdJRyAhM/9o\ninVvANZWHq+lyKiYbJ4Dy3kGx5m+oRzfGBGrMvOuiDgAuLtzw5l5XUTcDBzOOF33ZOa5U+x7XymD\nG/cB36tOL4uLPg44Eng68AyKZj6PH28999RqE/Z1X7VnJvuWwz7AvmWQYx9gv0z2pWjzNtXXajrF\nOmfSxepCpG8u5oPJXDCQIUmT2zeT5zYaPLdMGW9393pL2d3rLWV3rw9O8PveiuCOep076nUur0yP\nTA5otTis1eKwZrMYWq0d7mKre/6W9Y4h4KUjI3yq0sPHRAGMFZk8oZ29XDbNWpW5LVAxBrxm+XIe\nqtV4uFbjBwMDPHMemm9sAc5fupTvTlITo1uXDwzQhFkJvkx03ra+VuNDS5dWm9x8mVnuXrW8KT3c\nfhwR75rN9au3TJaJ0f5gnUTRZOFiiuvS3wZ+1sW6fwgcERGHAHcAL+OxTTwuAd4AfCYingk8mJkb\nI+K+SZa9BDidorvT04EvAJTz3p6ZjYg4mCIL4cYu9nOXVRYX3VgO32xPLzM3ngEcT9Es5RiKbmK7\nKvT6UAQPRXDzJPMMVbI59gD2qPzdtwx07AscmLmtANlEzUhmEsDonLd6QLWpgSRpVzVRd68PR3B7\nrcbtZbv+DbUat9Xr3D1J+/52cKN6F3efdlCj7Pb1sGaTfecwZV6aC785NsYg8C9DQ9xbfgf2bLU4\ntPx8H9pqcUSzyQFTfLYHgReOjfH5JUWDjK8NDs55EOPWWo2/XLaMu2ZQTL9TLZPnNRqzVi+nM4Cx\nMYLPDQ3xtcHBaqBoDHhjeZ2inbCYg6MTXrRm5qcAIuIPgOdk5lj5+KPAt6dacRlMeAPwFYrg3icy\n89qIeF35/Mcy84sR8eKIuIkiqHjmZMuWq34P8NmI+F3KLlbL6c8Bzo6IMYovx2sz8+Hu34rFo8zc\n+HI5ABDDw0MUgZ8DKGps7EWRXHEARZ2SA8u/B8DUx7rRCO7//9m78zi5yjLv/5/rVFV3OjtZOxtJ\nCGGHsIMImpFlEBBUGBlUBsEVBJ9ndFRmnJ/bOCMuo7Ko44AL484oo/DgKCBElE32fYdgCEkISyBb\nd1fVuX5/nLs6pytV3dVLdVd1f9+v13lV9amz3NV9uuqc61z3dQMv9/HPFbmzszt7FYvsFcfsUSwy\niaEv8DkcgYux/EFST42SjaFaGyLSzCa7b1c8FJI6G6tLwY0wKsqq8HOlu9MvRREvRRF3pOZNChd/\ni4vFpCtKqL0xWZ+Z0qAMeEs+zwn5PJ0kwYiBZiIclQpi3JvJ0Al1qzHxcCbDv7S1sTn1v3lcVxcH\nFAr8S5UhVyu5YMsW9igWKVDjHcx+WG/GLdkst2azPJrJ9PgcybhTNDvXly9/fIh3K2OM9XVSbmaP\nAYe5+0vh52nAre6+6zC0b8iZmbv7yF8RNSlbsSJLUkx1Hj2DG/PL5g14hLeFYei5veKY3QsFpleo\nldGIGuFCu5F/P4PVn9/vUP8eFMAQkbGmk+SO71OZDE+H6dkooqsfn8VTSoVEywqKzhyDI6U0wjmC\n1M+Hx49nVaiN8f9t2cJBxaEfIPGuTIYvtrV1/w+2uXNeRweHFwrEwDkTJvB8P7IzTu7s5IyuriFp\n2wYzbsxmuTmX4/EqdXn2KRS4H/b0o456eEh22ofRfs1nZn7JJZcM2/7OPfdcGun3WUvw7QLgbjO7\nkSRw+Ubgs/VslDQuX768QFKfZHW1ZUItjglsy+iYBt09SGaxLdCxGNi9fP1nQ2Gy34SfZ4YToInu\nZNzJkhy4pec5kqrrLSQf6NPcmRHHzIpjFSUbRRolI0NEZCxoBZbGMUvjuHt0lCLwXBQlQY3w+Ewm\n0+OucNqrUcSrUUT5FUuL+7bgRsjemB9GShmuURJEhtKBxWJ3EOPebHbIgxi3ZLN8ddw4CuF/bUoc\n8wbW7jIAACAASURBVNmtW1kSul5HwPs6Ovh8P7Ixftnayn3ZLF/bsqXPZdeacW82y5r77+eZe+4h\nn8nQUiyy53778aODD666nrmzT7HIO7q62LtY5KSTTnoEjTo4apjZVOAykvIETtKr4gmSMhQLCb0m\n3H3DUO+7ltFJvm9mvyUZ4tSB891dQ+JIVaGP26YwreptWVuxYhrweuANwBEkdTp6hHBrLThayfxQ\njGmXUM19YRwPW9VoGXojEchQFoaISCIDLIxjFsYxfxXmOUm/96czmaQbSibTXXujWtZGlxnPhABI\neqQUc2emO3PjmPkha2NeCHBMa/K6GyNRbFyGzz6FAv/Tkozf91CVTISBujGb5cJx47q7ZcyKYz6/\nZQtzy85NDiwWOSKf54/9KPb5ZCbDn7JZDisU+ElLC7dks0xz58h8ntcXCtyQy/G/uVzyv3rffXD/\n/fDBD3av/9Cll0JrKyxbtt22P9zRwcGFAjuEdp599tm7xXGsE6oh1ACfJxcCv3H3U8wsS3IT+1PA\nde7+ZTP7JHB+mIZU1e4kZrbE3Xur3VjTMo1mtKcWNTtbsWIicCjbghqHAuN6XakfWsOwWPPC3Z+5\n4fnsOGbiILfdAB8kwOjuUlLS1+96KH8HCmKIiPRfTDL863NlgY3nQnZGf00KI6UsKRa7p/YmC2w0\nynmCDL0twDsnTiQ2w9z5yaZNTBjkNovAz1ta+HlLS/eoHvNCAGNmlfOSV8z48IQJbOrnsbZTscjT\nfQVfvvtdeO97t5//ve/BWWf1mHVkPs//6ejo/vkLX/jCSbfffvtV/WrUII32az4z829+85vDtr8P\nf/jDPbqTmNkU4B5336msXY8CbwyDdbQDK9x9t6FuT2+ZGP9mZhNIRgO5E1hD0p1kDsnd8hOBjcDf\nDnWjZOzy5cs3AdeHqVRwdGeSYWInkPQeSU8tJEGONpKi7FOAebgvAJZi1uNMqdOMpzIZniq7+wMw\nwZ1ZIaAxO9xpWlIsskDZGw2nr4yMoQo86IRTRGRgImCWO7OKRfavMFLK6lRQo/R8XZViogAbo4j7\nooj7sttOXSe4s1MqqLFzHNMex2Ou3oaMvPHATnHMk5kMbsajmQwHDKJLyVozvjFuHA+njvdFxSKf\n27q1O7Ohkh3cOaujg4vaqpemy7pzSlcXP2vd1nmrzwAGQLVlKgQlTxyiWhvSu2gIRqgZhMXAejP7\nPrCMZGTT/wvMdvd1YZl1JLUUh1xvo5OcamY7kwQp/pWkXwvAsySjk5zn7k/Xo1EiJb58eRfwcJj6\nxVasGA/sS9IV6hDgdcCO1ZbfnE5vTcm5s2Oouj4vjml1p5WkT29LeHQzYpJxxmNgvDuT3JOhZcPy\nMrTq3bWktG1lYYiIDK3J7kwuFtm97CIvD6wJQY309FyVuhubzXggm+WB1IXe+BDY2DkM/7o4jpkb\nx0M+AkN/KSg++u1SLPJkOIdcOcAgxibgv1tbuTqX665/AbB3ocD5W7cyqYZtHFko8KdCgbuzlY/6\nghnL83kez2SqLpP2xnyercCfq72fUOPibzs7ObWra7sbf9///vc/MdxZGDIsssD+wLnufoeZfYOy\nbiPu7mZWlxPpXo9cd38S+EI9dixSb758+RbgljABYCtWzCTJ6khPO5NEEyuGrfPp7I0Bagn9fOeE\nvr27hRFYpgzxBfJYu+CuVyBDAQwRaXR5YIsZ6ZJ8rcAU96bNHswBO8YxO5YV/ivV3Sh9Fz+VyfBU\nFLGxwl3ILWY8mM3yYOriLBtuRiwqFlkUHpcUi4PuRlorBTDGhvRxu6qfd8i3AFe1tPDrlpYeAbvI\nndO6ujilQnCgGgPO6+jgIxMmsLHKsfehiX0f/R/q6GBGHLN7scgZEyfCgQfCZZfB+963baFLL6X1\noIM4s6ODN+fzTdW9azSo52fLY489xuOP9zoS7nPAc+5eGnH7F8A/AmvNrN3d15rZHOCFerSv7oFp\nMzsW+AZJPajL3P1LFZa5CHgzyf/we9z9nt7WDcO8blf11MyOBr5I0sWgC/i4u99Y33cozcSXL18P\nrCfJJuoWRlSZRRLMWAwsBfYDDgAWDHa/XWasDumzd6bmzwvBjN1D4dH2Jj75HClDHcjQyaYMhQJJ\n3+SXo4iXzXgpPC89bjUjT/JFFZsxJY6Z6s5Ud6aHLm1z3Jkdx0xpsn7/MnBbgBejiBfNeDEUtX4x\nHDObzdhKkn2w2axq0cwoHEel0T8WhG6RC5r4WDKg3Z32QoHXFwpAEthYHwIbT6YCG69VuHgshMKj\n6ZR5c2dRHLNXocDexSJ7Fgp1CWroO2XsWJAKYvylxiBGB/D/Wlr4n5aW7QIOuxaLvK+jg10HMJrH\ndHfO6ejgS710K+nLcWFUojVmSVZIKN658NJL6chkyBSLvH7ZMt62ZAkTw7Llrr322kuvvPLKrwy4\nETJidt11V3bdddfun6+55poer4cgxSoz28XdHweOAh4K0xnAl8Ljr+rRvroGMcwsA1xC8qZWA3eY\n2VXu/khqmeOAnd19qZkdAnwbOLSPdc+nctXT9cAJ4Ze6J/A7kqE8RXoVRlRZF6bb0q/ZihXTSfp6\nLSM5ntoqTDHJdUsh89prU3Zva1u+MYrYaMZrpQ//ClZnMqzOZLg2/JwJGRuzwhCxs0t1OsLFzA7u\nFfv6jvWMgaEKZKS3MdZ/p1LZFuCFKOI1MzaGi8mNIUjxYhTxYghUvGrWXYitFmt7OeEdF7K42kNw\noz0MRTk7jpkZhpqW5rCVpLtEaXohFbB4MQQqBis24+UQ+Hig7LVJoZj1rPDdMtWdNnfGuSfFpcLz\nNpLjrjS/EWtMGKHmRqHA61KBjRfNuoMaK6OIlZlMxRHOPNWF9GqS4M8+YXSHQ/P5IQloKIAxtqSD\nGM9FETHV/3c6gd/mcvyipWW7Qrfz4ph3dnZyeKEwqKDjsvB/MRBnpYpyznZnahyzIYpg2TLOXrqU\nPYZ4CFlpWucBPzazFuApkiFWM8AVZvZeQrJBPXZc73Ofg4En3X0lgJn9DDgJeCS1zInA5QDufruZ\nTQ2VTBf3su6JwBvD+pcDK0iGfr03td2HgTYzy7l75fCgSA18+fKXgBvC1G8hy2MSIcNj0bPPfnLc\nvHkHPpnJbBfcKJqx1qzqBU3WnZmpoEZ76oKmPY4r94cZI4YyI0MBjLGrSHIRtCaKWBsKDXZPZhXT\n1+uto0q9HkguvGa5VwxwtMfxgC9AnaS7QocZneGxi+TMJEfSRS4HTFQQZTudJIGK50OgIv34yhAe\nP5E7E0LAobTVLVAxE6FkYxTxaBTxaD/3VQpotIV6T1NT06zSiF9xzOQRzvQwYKY7M1OBDUjqDKwM\nQY3S/9IzZUVEYzPuzWa5N5vl2+PGsW+hwOGFAofk84wfYHtGYlhwGTlT3JkUx2yMIjpCcLt8FJE8\ncG0ux3+3tPBy2f9qexxzamcnywuFQWflrjfjH8cP7Mh9f0cHb0llVkTAwYUC14YhZO/IZGoKYjz0\n0EM3XXzxxR8YUCOkJiP9+eLu9wEHVXjpqHrvu89zD0tGd3gXsNjdP29mOwLt7v7nGrY/D1iV+vk5\nkgKLfS0zD5jby7q1VD09GbhLAQwZaSHL4zXgvjD9AsBWrGgj+cd/PXA4yag/s3rbVsGMNZkMa6q8\nPjUUMJsfUod3LxTYaQyNrjKYE8aR/iKQ+iiSXNhtNmNTyJwoTa+ascGMV6KIDeHu9QtRVDVzqj/M\nnSnuTAvdQ6a5My2OmR7mTXDvLg4MbGuLGetD0GRtuPjt6KU9cR+Bz1JbMiQnopnSFLK60j/HqYBF\nZ9h2LSa4Mzl0VSgFVNpDmv7sURoQ7IDuv08pSFF6Xn5h0h85d2aEDJsZccyMELieHo6Z8anHcVAx\nYJAnDG2aybAqivhLFLEqDHHa27HU6/s1o8OMDVD1+weSY2FescjscNxPTx3/k9yZGKbhDnxNBPYq\nFtmrWIRwcbYFeDST4YFslvuz2e6CjJB8196Zy3FnLkdu3Dj2LxR4fT7PQYVCvwMaCmSMHUaStbAx\n/Lw+ipgZLvaLwB+yWX7S2soLZZ8RM+KYU7u6ODKfH7L/jUsr7CctcueIQoE/lI3U9++bN7O0QveV\nA4vF7qzhu7NZztDoIzLCavlf+RZJqvybgM+TBLS/RXLB1Zdaz15q+XS3SturVPU0dCW5ADi64obM\nPpv6cYW7r6ixnSJDxpcv3wrcFCage0SVhSRZG4t6TO6LMZvR2zY3RBEboqjHUC4T3DmgUODQfJ79\nC4VRn60x2BNGZWGMvCLwghmrMpnuDIhOM4zkQjybuijvIinktzVMW1I/lx6HUjbced4hdUE2MRWs\nmBEuPnfo54VatX6PTjIc5dooYo1Zd3CjNNVywexmbJdUPIS/l81mbA7B1fI7/IuLRQ7N5zm0UGBh\nHDdNPYYC8FL4fa+PItanfvdrBpFRkQlZdHNLGTMhSFEKWAxFzYocMMedOYVCj9tjMcnd2bXhPb0Q\nujx2sC1I0QHd/0ul5539OFY2m/F4NkuvpeBIuq1MdmdKCH5NDu99cgjQtIYgzbjw80SSrJ8JQ1g3\najywf2n4185O1ptxcy7Hn3I5nkgFNPJm3J7LcXsuR0v4Pj0sfJ/W2uWk/HtFQY3Ra2YYZhWSYrR7\nAI9GEd8aN46VZdl00+KYv+nq4ph8nlyFbQ3UejNuSwUnPrZ1K9Pd+adUZsaiOOa5ss+xr2/ezJIq\n9Tf2KRTIuFMMmYGvmvVanH7Dhg3rzj///DdWXaAOzGw5sHw49znSxvJnSS3nWIe4+35mdg+Au79s\nZrX+r62mZ1HEBSQZFb0tMz8sk6swf3V4vq5a1VMzmw9cCZzu7s9UapS7f7bG9osMqzCiyiP07HLV\nzVasmMi2IMeSHpP7TlT439xsxk25HDflckThBHp+qP6+uFhkp3Ay3Yh9noeLRiMZOVuAx0N691+i\niGfD3eNqRQuHw9RUF632VPet3urS1IuRpChPKRbZtcLrnbBdYGNt6kJ1ML/HTOpCsjS0dJHkTn9X\n6F6yqY/aH6XU/Z8CU+KYvYpF9igUmBvqMbQArWWPg/39FthW/HJzaOPmENTKk2SYWAgWlO6OvBwC\nFS+Ei/tXzGrORClXClTMCdPc1PNZQ3gR3l8RyV3i2cUi9KM/e5HkOCsFN14ty2IqDYn6fD8yPUrb\nWjeAYFBbKng4oTxDhSRw3xY+y0uXY56eQhvTr6Uv247I59mzUOC+bHa7LlxdZtyay3FrLkfGnT2L\nRQ4uFDgon6e9H98fvX3XjOWLktFgxzjm1vD8jyEgdk0u1+Nzckocc3JXF2/O52mtQxteKjuGlhWL\nTHXnfR0dXDZuHECPIrcAp3V2Vg1gQBL0W5wK0Pwliti7l8+R008/vX2AzR+wcFN6RelnM/vMcLdB\nhk8tQYyuUGQTADObSc/P+97cCSw1s0XA88CpwGlly1wFnAv8zMwOBTa4+zoze6mXda+iQtVTM5sK\nXAN80t1vRWSU8eXLN7Gt8m8PtmJFBtgR2AXYnWRklSOBOaVl4lJ3lEyGO1LrjnNnUahgPzsUFZ3l\n20ZGaLbuKP0JRuiEcXhtBB7OZnk4k+GhbJanyvqlDzVLXdiUT5Pd2aHUtz8EKGbG8YD7v4+EVioP\nSZkWk1yIlh4LZt3Pu6fwNyjVPmiFmu4MFkkCBq+FrjClgMrTUcR92WyPrjmvRhE3RxE353rf8rjQ\n9WaHkOEyNdT7aQmfRVtTAYpK3YQG2mWiP0oZOXMqBCtGW7HVDMkFzHh3cO81a+hlS0bhejGMylMq\ndPtKFLEZuovhDuZ/vhQAWT/gLQyNohn3h64ol40bx44hoHFwocDSYnHAwbjBfH/pu2/kHVQo8PPW\nJDRxZ7bnJ0GrO2/r6uKtXV11/Z5ZHMdMdGdT+Bt/bPx4zgl1Lp7IZLbrQvKBjg6OrzK6SNqCVBBj\nVR9BDBke0QjU6WoUtXzPXgz8DzDLzP4NOAX451o27u4FMzuXZJSQDPBdd3/EzD4YXv+Ou//GzI4z\nsyeBzSRVTauuGzZ9AZWrnp5Lclf6M6no29Hu/mIt7RVpZr58eRF4Jky/g+6iovsCbwvTnlTovtVh\nxqPZbNVib9lwR7Y1lebblkr/Td8Jm5i6MJwWLj6a4VRJWRhDbwvJ3Z7S0IdPZTI8V6E4ZSVTw4V5\nqUjlRHdiwjBAZt0X3y0kF1fjw93X0sVWW+rnsfsVn4go+x1UOtYHePxngMnhc2A+9Dip3UJyEn9b\nLsf9mUzNRVE7zHg+k+H5AbVo6OyQCujODM9LAYsZTRjcrTcjGdZxeh+ZHjHJsfFqGOXn1RAEK2V4\npLu2dIQMmk1h2gL9GvVnOP0lk+EvmQy/aG1lShxzUKHAQYUC+xYKjKvTPgfzvaUuLvWxSxxzcD7P\nn8sCBfsXCpzT0cGsYTjXaAXO6+jgi2F41fVRxOfGj2fvQoG35PPbBTHeUOMoKPPLRl+p5rTTTtth\n06ZNA2m6SM2slg9AM9ud5I4uwO/TQ6Q2GzNzd9cntYxJoeZGKVNjb2C/MFUqjjskWkNQo3RndYdQ\n5K00r1TwcALDH+zQkKq9y0N3TYkopN+XLogNuutSbAVeDGnlz4YhDZ+JIp6vIWBh7iyMY3YtFllY\nLHZnFfTW11aaTwysjCIezGZ5NhwrW0IB0c7QNaXTbEi6EZVG7ZhAzwyc8aXuKu5g1qOLwWTfNrT1\nrBCkGMo+6jI0SgGQTenARphKXYY2kwQ/St2FYFsQrzSv1J2o9Fr6qCuSdBvZFIIrpZo0/akNUm5W\nHPO2zk4OKRSY3gSfbQpoDM4W4JvjxnFXNkt7HPP2ri6OGORwqQNxWzbLRePGdWdk9OaUzk6OyueZ\n28vxeWM2y9dDYOSN+TwfSw3DmhaCGBsG1uqhM9qv+czMv/vd7w7b/t773vfSSL/PqkEMM5tWPis8\nOiS1MerYrroZ7Qe0yEDYihVzSDI2lrCtmGipuOgODEN8YVzqbmepOGKpr2g2pLeX7q63VXje3wuO\nwaThNhInSdFeE2ohrAsjbWw2Y2M4Ed8aLtjSJ/WVnsfQozjmUIzSUS7jzpJikT3DtHs/iuPJ6Ock\nKZkvh7oUpccukgvLAkmmTXmAIl0fYaDDyopUE5PUGVgVRawOdXuezmR4doB1Z6bHMe/s7OSIOtVE\nGAoKZIwOr5jx85YWfpfLdXcb7M1Ht25leWG7UtAA3JTN8tUQxDgin+fjFYIY//AP/3DoY489dvvg\nWj00Rvs131gPYvTWneRu6D7v3RF4JczfAXiW5AJHREYBX758DVVGzgtdUlohuW4I03hgKjATmBae\nl6bpQDtJLY45UFsmbYclo1GsqrG7QblsuIBJd2WZER4nhyBIa+jv30LSDaGVpGDdhlQ682vhoj8X\n+rW3ApPCNiaFtPnJXn14w3JdJCe/L0YRRtKvvzU8looZtpJ8GMdsqy+wKQQhSn3/y+82biEZjWZt\nFLG5QU82I3d2jGOWFIvJFIrJNupJu4w8IxkOc2IcsyP0qwClSL1EwEx3ZpZGMwmKwOoQ0Hg6dJl7\nOpNhSx+fyS9FERe3tXFxuCA8MJ9nUairMi88TvLqI9WUMlJeC11yNqa+N7pIRlQpkPw/5ULXpwxJ\nnZtM+G4rZdVF3nMI5u5HdzJm3fMqLufevZ3t1i2bX57x0h+l7oMFtnUnzJB87+dIvj8b81tw5O3g\nzoc6Ozmpq4v/19LCDblcr+cMX2trY/7mzexcoc7SxtR66k7XGMZysLFqEMPdFwGY2aXA/7j7b8LP\nbybpWy8iY4AvX+6QdE8GXurPuiEAMoUkmFEKbMxlW4Cj9HwBDG7010KpT/VgNtIP2VRAY1JIVS+N\n3lAKPrxkxoYa6wBYyARppP7emZDpYu7EoRhkqZJ/9xQCPjND0GhBscjiOGZR6BqigIWIjFYZthXW\nXR7mOckQ0U+HgMat2Wyfwfk7cznurHNbG0FUKegR5mXYlhGYZ1uwogB9FoI1T4bhnRbHSV2W8DjD\ntw19PX2Euq02ijnuvL+zk3d3dnJjLsed2SxrzZjuztOZTI8AxUcnTOA/N22i3b1HlupjqfOZJRUC\nzF//+tfPaJQsDBn9ains+Tp3f3/pB3f/XzP7Sh3bJCKjRAiAbAhT1Vo6IdgxjSTrayFJUGM+yY2j\niFC/EZhUYZocHof1xkDBjJfNGKp+dYMJXoxz7x4OtD3UGZnk24YhHB/u6JVORbzCBMnJXbowZgt9\nn/Cl1xURGeuMMJRtocDrCgXe1dkJJBl/D2Qy/LK1dbuhW8eKUjC8hyEI3LsZG4GNmQzP9rJcqzvT\nQ4bmdPfumlylwMfMkM05mv86bcBx+TzHpUYjceCaXI7/HLctcfYDEyfy75s2sXMIYrxsxm2pgqC7\nKktORlgtQYznzeyfgR+RfDa/E1hd11aJyJgSgh0vheme/q4fgiDjSAIhpYyPeWGaT5INUuoGMyEs\n2xamDmB9anoReI3k87GFJEAyPUwzUlOtBedjks/M50huKo1P7bsNGI97G2Y5oDvwUB6EKBUmTNcD\nmBiCF1N7STuuNwUvRET6NtWdIwoFjgj1BjaYcXVLC79obdx8tfY4ZnocUwzBh/RwzTF0zy+WvVY+\nv5S1N1AWusCUpihkB+ZJsjZqqfUASfHg5814vpcMyUwIdMwMQY1Kj4NKG21ABhzf1cVPW1p6jCL1\nyQkTODqf562dnfx3a2v38NULi0V2KetucsUVV3zhhhtu+K/hbLeM7e4kfY5OYmbTgc8AR4RZNwGf\nU2FPERnLwkgvpcDGdJJgx0SSQMXLYXoBWO3Ll1euktVzexnoHiq3rk455ZTzTz/99C8ARFGUadai\npiIizS4GHslkuDeb5YVQhLnUvaLAtvoW+dRFez7UwXi5TgWY0+bFMcsKBfYtFtlrEIWY010RewQ9\nQreR0mvltTuy9J1mWQReDV04XzbjpSjipdTji+F5xxD9riaWghpVAh07hC4yja78u38rSeDi2T4y\nhT63eTP7pYIYN998839fcMEF76hHGwdjtF/zmZl///vfH7b9nXnmmQ1V2LOmIVYHvHGzY4FvkHz+\nXObuX6qwzEXAm0lqFL3H3e/pbd0wasrPSVLOVwLvcPcNYf4vgQOBH7j7eVXaNKoPaBGRRjFr1qwd\n58+fv/vpp5/+bzvvvPP+pfkKmoiIDF5MUoPppTCSz8upi/iXzXgliugk6f5YoGdQpEDtGQwlkTtL\n45i9CgX2KhbZrVhkwpC/q/pwkguNUmAj/Xt6ORTfXm/GqzXWsepN1p1Z7uwaRuHaq1DodejS4dbX\nd/AaM746fjxPVghmHJDP8+mtW3tkACiIMTLMzH/wgx8M2/7e8573NFcQw8xurDDb3f1NfayXAR4D\njiJJpb4DOM3dH0ktcxxwrrsfZ2aHABe6+6G9rWtmXwZedPcvm9kngR3c/XwzGw/sB+wF7KUghojI\n2HLUUUe9Z/HixcuOP/748zKZjDJMRER6USqiuTWMelXK7lgdRTyYzfJQJtPrELLmTnso6LxjHLMg\ndG/s7gZJ0vWxv0Ogj6RO4EUz1oegxvoo6vl8gNkvc+KYAwsFDggBoJahb3qf+vOdmAfOmTiRF8qC\nOmd0dPD2rq7uIMbKlSvvP++885YNZTuHymi/5hvrQYxaamJ8PPV8HHAySQC3LwcDT7r7SgAz+xlw\nEj2L+50IXA7g7reb2VQzaycZvrXauicCbwzrXw6sAM539y3AzWa2tIa2iYjIKHP99df/IDz9++HY\n31e+8pVbp0yZMnPOnDlLQBkmItK7Ruu/Xhp6dRzJUJyUPsOKRf4mn6eL0NUlk+G+bJanoqhHEWo3\nY40Za6KIP/eyn5ZUjaeJIbgx0Z0ZYTjZOe7MjWOmjGB9p5JWYJ4786oUroxJuq6sLw90pB5fq5DN\nsSaKuLqlhatbWhjvziGFAofn8+xbLNY1yDOQ7yUHvt7Wtl0AA+Dnra28Lp9n7hC0TQYvGoLMoWbV\nZxDD3ctHffqTmd1Rw7bnAatSPz8HHFLDMvNIhl2stu5sd18Xnq8DZpc3uYa2iYiIDMrHP/7x1w3H\nfmbPnr0wl8u1futb33qsdBGkgIlI8+nv/+1IBz1agGXFIsuKRejqYiPwQDbLw5kMD2UyPBNFNRXs\n7KpxNK/x7t1FI3ctFtm1WGRGAwQ20iKSgM8O7tsVtyzpAFZGUfJ7ymZ5IJPpUY9jixk35nLcmMsx\nwZ1DQ0BjWbFY093lWgzmO+LXLS3cnBqJZEmxyIuhq02HGZe0tfFvW7eSz+c7GzULQ0a/Pv9XQq2J\nkoik5sTkGrZd639PLZ9N6dEBt+3A3c2s3/+lZvbZ1I8r3H1Ff7chIiIyHNatW1caNXBYzuXf8pa3\nnDd37tylxx9//HkKmoiMnEYLekwCDisUOCyMsNIJrI4i/hJFrIoiVkcRm0LXlE1mbAE2mdU8MskW\nMx7JZnukbE+NY5bEMUuKRXYKj7MaLLBRbhywWxyzWxzz9nyePPBQJsNd2Sy3Z7OsTd0932zG73M5\nfp/LMTFkaBw2iAyNwX5WdwI/So2Y89ddXXyoo4Ono4hPTJhA0YwHs1lWmzFrUHsaema2HFg+ws0Y\nViMd6BxJtQT87mZbAKFAUkzzvTWstxpYkPp5AUlGRW/LzA/L5CrMLw3rus7M2t19rZnNIan+3y/u\n/tn+riMiIjIWXH311ReHpx+p977mzZu39P3vf/+Fc+fOXdre3r5zvfcnMpoN5AJ2MBdBrcBOccxO\nVTISILmA6CAJZmwyY3N43AisiyLWhODHmihia4W2bIgi7ooi7spuu2RpCSOBTEt3U2Hb0OMTwjDl\npWmiO+NhxEYMyQH7FovsWyxyVmcnT0cRf8xm+VMu16PLxqZUQGO8OwcVCrwu1NHoayDeoQo0v2JG\nPvV3eH9HBxGwII6Z7s4L4bUXoogPvf3ttQ41PyzCTekVpZ/N7DMj1hipu1qCGLu5e0d6hpnV5MKm\nUwAAIABJREFUctDeCSw1s0XA88CpwGlly1wFnAv8zMwOBTa4+zoze6mXda8CzgC+FB5/VbbNsRuS\nEhERaSKrV69+AjhuOPYVRZGddNJJHz3rrLO+CsouEYGB/x/UGvwwoA1oc2dmL/ty4GUznowiHstk\neDyT4YlMpmJgo8uM1ZlM993NmtoRAhkTQpBjgjsZtk/zttTU28+EYVTHAVPcmeze/Tg1FDjdwX27\n4INBklnS1cUZXV08EUX8KZfjT9ksL6YCGlvM+EMuxx9yOVrdOaBQ4PUhoDG+9Durw2fYDHda3LsL\nuv4hl2Npsci/l9XImN1L4EqGz1jOxKhldJK73X3/vuZVWffNbBsm9bvu/kUz+yCAu38nLHMJcCyw\nGTjT3e+utm6YPw24AtiR1BCr4bWVJNluLcArwDHu/mhZm0Z1pVoREREZWTNnzlywcOHCvU877bRP\n77LLLocoYCLS/wuumGS4z6cyGZ7KZHg6ing6k2FjE124tYXgRjZ0gYnYFgiJUo8AT0VRzcPeHprP\nc2xXF1PdKZL8ropmyWPq50wIpLSGxx3iuDsIUs33Wlv5dapLibn3KOh6bFcXd59++uJ169atrKmx\nI2S0X/OZmf/oRz8atv29+93vbqjRSaoGMUJXjbnAj4F3sq0uxWTgP9x9t+Fq5FAa7Qe0iIiIjC1H\nHnnkGUuXLj3wmGOO+UA2mx2J0RtFhly1oMdmku4Mr4auKd3dVNjWbaU0bQxTpYyOsWqcO9NCd5wd\nQubIOJKLvM1htJl7s9sn67e4877OTm751KeOveeuu3437A3vp9F+zWdm/uMf/3jY9veud72roYIY\nvXUn+WuS7hrzgH9Pzd8I/FM9GyUiIiIitfn9739/Ocmw8+cNx/6+/OUv3zxt2rS5s2bNWjQc+5Ox\nqdqN1vHAoipDoFYLfBSBLSQX6aW6HKWtl9bwsokqz0tZCTGwlWTI1VdDUOU1MzakpkIDBk86zHg+\nk+H5fqwzr1jk/K1bWejOrcoskwZQNYjh7j8AfmBmJ7v7L4evSSIiIiLSqD7xiU+8fjj2097evjgM\nL/yIuuRILaodJxEwEZhY9no9awo4sAl4LYzQ4iSBj/Rjj3lhmR7z3OkMgZfNZjyeyXBTrvZxS/Yo\nFCiS1BHZEoa6zQ/gPV+wZQuT3fnPSy/9yN13393wWRgy+lUNYpjZ6e7+Q2CRmX00/RLJ6KZfq3vr\nRERERGRMWrt27TPh6bDczj7ppJP+fvbs2YtPOOGEYclokZFXz2FsjaRQ3yR3GGQQLt3Oj23dCsBa\nM27J5bgll+OJTKbiequiiOX5PEfn8yyM4+7AystRxCtmvBIe82wrwjrBnQvb2rq38aGtW5msIGJD\niqKRGnNn5PXWnaRU92US2xfvFREREREZNX79619/PTyt+/DC7e3ti88555zvzJ49e/GcOXM0vHCT\nqGfQoz/bcXfa3Xl7Vxdv7+riBTNuzeW4OZvlsVQ9i41RxNWtrVzd2srSYpFjurp4QwhoLASo0C3n\np6minlPimDfk8wBcc801l6SG4BYZUb11J/lOeHq9u/8p/ZqZHV7XVomIiIiIjFIhy+SYeu/HzCyT\nyWSPO+64c973vvd9o977k576E/ToV5ZH2bKz3Dmpq4uTurpYZ8YNLS38PpdjfepO/ROZDE+0tXH5\nuHH8dVcXx3d1Mb2sfVuAK1u21QZ+d2cnEwbQPhkeY/lv0lsmRsnFwH5l8y4Cahli9Vi2DZN6mbt/\nqcIyFwFvJvm/eY+739PbumGI1Z8DC9l+iNV/BM4iqd/zEXe/tob3JyIiIiIy6nhyFZ0HLgxTXc2c\nOXPBTjvttN/JJ5/8yd122+2weu9vNBlI3ZfSRWz6YrYdeGdXF+/o7OT+TIbrWlq4PZvtLjK6yYxf\ntrbyq5YWjsjnOaWriwVxDMDD2SxdYbn5xSJHhiyMe++997r/+I//UDcraRi91cR4HXAYMDPUxCj9\nd0wiCSz0yswywCXAUcBq4A4zu8rdH0ktcxyws7svNbNDgG8Dh/ax7vnAde7+ZTP7ZPj5fDPbAzgV\n2INkRJXrzWwXd4/78wsR6S8zW+7uK0a6HTJ66JiSoaTjSYaajimpZv369auAVcBV/VlvoMfUm970\nptN32223w970pje9p6WlZVx/1292vQU+MsB+xSL7bd3KxijihmyW37S0sDZkZxTNWNHSwh9yOQ4r\nFHhHZyebUsGQ5zIZCtRw0ScjRpkYlbWwLWAxKTX/NeCUGrZ9MPCku68EMLOfAScBj6SWOZFkSDDc\n/XYzm2pm7cDiXtY9EXhjWP9yYAVJIOMk4KfungdWmtmToQ231dBWkcFYTnIcigyV5eiYkqGzHB1P\nMrSWo2NKhtZyBnBM3XDDDT8EfgicPcTtqeiCCy64adasWYtmzJixYDj2N1QmxTEndXVxQlcXd2Sz\n/LqlhYdD7Qw34+ZcjpsrjHryjsmTueypp/7y6U9/uu5dn0T6o7eaGH8A/mBmPygFE/ppHkkktuQ5\n4JAalpkHzO1l3dnuvi48XwfMDs/n0jNgUdqWiIiIiIjIoJx//vlvGI79zJkzZ0lLS8u4iy+++MGh\n3G4GOLRQ4NBCgccyGa5oaeHOPoZsPbtY/Pf3DmUjZMgoE6N3W8zsqyTdNErj7bi7v6mP9Wrt2FXL\nb98qbc/d3cx6249GVRERERERkaaxZs2ap8LTul6lfhywFSv2m/Dqq9/ePGVK+c1mAPK77LJvPdsg\nMhC1BDF+TFJI8wTgg8B7gPU1rLcaSKdaLSDJjuhtmflhmVyF+avD83Vm1u7ua81sDvBCL9taTQV9\nBD5E+s3MPjPSbZDRRceUDCUdTzLUdEzJUNMxNXI29/7ymQZnDk9LpD+UidG76e5+mZl9JNXF5M4a\n1rsTWGpmi4DnSYpunla2zFXAucDPzOxQYIO7rzOzl3pZ9yrgDOBL4fFXqfk/MbOvkXQjWQr8ubxR\n7j52/9oiIiIiIiIiTayWIEZXeFxrZieQBBV26Gsldy+Y2bnA70i6YH3X3R8xsw+G17/j7r8xs+NC\nEc7NhChftXXDpi8ArjCz9xKGWA3rPGxmVwAPAwXgHB/IWEUiIiIiIiIi0pCsr+t8M3sL8EeSrhoX\nA5OBz7p7v4ZOEhEREREREZHBMTP/5S9/OWz7O/nkkxuqR0PU1wLufrW7b3D3B9x9ubvvDywZhrbV\nzMymmdl1Zva4mV1rZlOrLHesmT1qZk+Y2SdrWd/M9jGzW83sQTO738xah+M9yciq1zFlZovMbKuZ\n3ROmbw3Xe5KRVc/PqfD6jma2ycw+Vu/3Io2hjp9TB6c+o+43s1OH6z3JyKrjMXW0md0Zjqc7zeyv\nhus9yciq4zE1zcxuNLONZnbxcL0fGRnVjo+yZS4Kr99nZvv1tW6tx6Y0pj6DGFV8dEhbMXjnA9e5\n+y7A78PPPZhZBrgEOJZkpJXTzGz33tY3syzJ2NMfcPe9gDcC+Tq/F2kMdTmmgifdfb8wnVPPNyEN\npZ7HFMDXgGvq1HZpTPU6ph4ADnD3/YBjgG+G7cjoV69jaj1wgrvvQ1LP7Id1fRfSSOp1THUA/wz8\nQ32bLyOtj+OjtMxxwM7uvhT4APDtGtbt89hsdGY2bFOjGWgQo9GcCFwenl8OvLXCMgeTXDyudPc8\n8DPgpD7WPwa4390fAHD3V9w9rkP7pfHU65iSsatux5SZvRV4mqQmkIwddTmm3H1r6ruuDXjV3Yt1\naL80nnodU/e6+9ow/2GgzcxydWi/NJ56HVNb3P1moLNeDZeG0dvxUdJ9nLj77cBUM2vvY12dqzex\n0RLEmO3u68LzdcDsCsvMA1alfn4uzOtt/V0AN7PfmtldZvbxIW63NK56HVMAi0Oa9gozO3woGy0N\nrS7HlJlNBD4BfHaoGywNr26fU6FLyUPAQzRe9qXUTz2/+0pOBu4KFxQy+tX7mFIR/9Gvt+Ojr2Xm\n9rJuLcdmQxvLmRhVRycxs01U/2AYX5/mVGdm1wHtFV76VPoHd3czq9Tu8nlWYV75+lngcOBAYCvw\nezO7y91v6G/7pfGM0DH1PLDA3V8xs/2BX5nZnu6+sf/vQBrNCB1TnwW+7u5brBG/ZWRQRuiYwt3/\nDOxpZrsBvzWzFe7+ar/fgDSckTqmwr73JBll7uh+NVoa2kgeUzIm1Po3r+UcSMfWKFE1iOHuE4ez\nIX1x96pfeGa2zsza3X2tmc0BXqiw2GqSEVZK5od5ANXWXwXc5O4vh/38BtgfUBBjFBiJY8rduwjD\nFrv73Wb2FLAUuHvw70hG2gh9Th0MnGxmXwamArGZbXV3FY0dBUbomErv/9HwObUzcNeA34g0jJE6\npsxsPnAlcLq7PzPoNyINY6Q/p2TUKz8+FpBkVPS2zPywTK7C/FFzbI3le1ejpTvJVSSFogiPv6qw\nzJ3AUktGh2gBTg3r9bb+tcDeZtZmSZHPN5Kk1sroV5djysxmWCiQZ2Y7kQQwnq7LO5BGU5djyt3f\n4O6L3X0x8A3gXxXAGDPq9Tm1KHznYWYLST6nnqjLO5BGU69jaipJ4eFPuvutdWq7NKZ6naOXjN2r\nuLGjt+Oj5Crg7wDM7FBgQ+gqMphjSxqYuTd/5oyZTQOuAHYEVgLvcPcNZjYXuNTdjw/LvZnkJD8D\nfNfdv9jb+uG1dwH/SJJ6dI27N13lWum/eh1TZvZ24PMko9zEwKfdXSNKjAH1/JxK7eMzwEZ3/9qw\nvCkZUXX8nHo3SZX2fJg+7e6/Hc73JiOjjsfUP5McU+lg2NHu/uKwvDEZMXU+R18JTAJagFeAY9z9\n0WF7czJsKh0fZvZBAHf/TlimNArJZuBMd7+72rphfp/nVY3MzPyqq8pjOfVz4okn4u4NEzQcFUEM\nERERERERkbFgrAcxqtbEEBEREREREZHGo5oYIiIiIiIiIiINTkEMEREREREREWkK6k4iIiIiIiIi\n0kTUnUREREREREQkMLOfmdk9YXrGzO6pstxUM/uFmT1iZg+HYU4xs2VmdquZ3W9mV5nZpDB/mpnd\naGYbzezism39q5n9xcw21tjG3cI+OszsY4N9z9IclIkhIiIiIiIiPbj735aem9lXgWpDkF4I/Mbd\nTzGzLDAhzL8M+Ki7/9HMzgQ+Dnwa6AD+GdgrTGm/Bi6m55DMvXkJOA94a43LjxrKxBAREZHtmNmm\nOm//GjObbGZTzOzsAay/3Myu7uc6s8zsmiqvrTCzA/rbjnoxs1Yzu8nMdL4iIjJCLLlafgfw0wqv\nTQGOcPfvAbh7wd1fDS8vdfc/hufXAyeHZba4+81AZ/n23P3P7r62wn5mhmyPP4fpsLD8ene/E8gP\n/p1Ks9BJgYiISHVe1427H+/urwE7AOfUc18p5wI/qNYkhuA9hztxg+buncAfGYN32EREGsgRwDp3\nf6rCa4uB9Wb2fTO728wuNbPx4bWHzOyk8PxvgAVl6/bn++ZC4OvufjBwCkmWx5hmZsM2Vdn/sWb2\nqJk9YWafHM73riCGiIhIP5jZvmZ2m5ndZ2ZXmtnUMH+FmV1gZreb2WNmdniYP97MrjCzh8Lyt5nZ\n/uG1lWY2HbgAWBL6HX/ZzN6YzrAws0vM7Izw/NjQ7/gu4G2pZSaY2ffC/u82sxOrvIVTgGvCOm2h\nz/PDZnYl0Jba3jFmdouZ3RXaPyHMPy7s/04zu6jUTjP7rJn90Mz+BFxuZjMq3TWr1k4z2zPMuyf8\nbncOTbkKOG1QfzQREanIzK4zswcqTG9JLXYa8JMqm8gC+wPfcvf9gc3A+eG1s4BzzOxOYCLQNYim\nHgVcYkldjl8Dk1LBEhlmZpYBLgGOBfYATjOz3Ydr/6qJISIi0j//BXw49PH9HPAZ4O9J7ihl3P0Q\nM3tzmH80SYbFS+6+p5ntCdyb2lYp8+GTwJ7uvh8k3UTK9umAm9k44D+Bv3L3p8zs52y7k/Up4Pfu\nflYIrNxuZte7+5bSRsysHSim5p0NbHL3Pcxsb+DusNyMsL0j3X1ruMPyUTP7CvAfJKnDz5rZT+h5\nJ2034HB37wyvfd3dbzazHYHfkpzoVGwn8EHgQnf/iSWZHKVzlHuBw/r+s4iISH+5+9G9vR4+j99G\nEqio5DngOXe/I/z8C0IQw90fA/46bGcX4PhBNNWAQ9x9MIGQUSWKRjQf4WDgSXdfCUkRWOAk4JHh\n2LkyMURERGpkSd/fKak+vpcDb0gtcmV4vBtYFJ6/HvgZgLs/BNxfadO17J4kSPBMKqX3R6l1jwHO\nD3epbgRa2T51dyGwJvXzEWEbuPsDqbYdShJwuCVs7++AHYFdgafd/dmw3E9T+3fgqtAFBCrfNZtQ\npZ07ArcC/2RmnwAWuXtHaFcnEIUAjoiIDK+jgEfc/flKL4b6FatCkKK0/EOQ1LEIjxFJIc9vl63e\nn8qU1wIf6V7RbN9BbEsGbx6wKvXzc2HesFAmhoiIyMCVnzSVLuCL9PyO7e/JVYGeNxpKF/Dl/YfL\nt/t2d++ronv5OtV+vs7d39njBbNlfay7pey17e6ahb61ldr5qJndBpwA/MbMPujuN6a2Vdf6JCIi\nUtGplBX0NLO5wKXuXsqsOA/4sZm1AE8BZ4b5p5nZh8PzX7r7D1LbWAlMAlrM7K3A0e7+qJl9maT7\nSpuZrQr7+TxJAOObZnYfyffrH0i6qrQDdwCTgdjM/g+wh7vXtTB3I7A6jk5y3333cf/9le65dBvR\n72QFMURERGrk7q+a2Stmdri7/wk4HVjRx2o3k1R1X2FmewB7V1hmI8nJXMmzwB7hhHA8cCRJgctH\ngUVmtpO7P03PWhG/IznJOw/AzPZz93vK9vMs0J76+SbgncCNZrYXsA/JicltJCeLS0K3lQnAXOAx\nYCczWxiyMU5l24lM+dlU6a7ZV0N7lrn7fdXaaWaL3f0Z4OLQ/WTv0K5Wki4w21WxFxGR+nL3MyvM\ne55U15Dw2X5QheUuAi6qst1FVeZ/AvhEhfkvAX9bYf5ats86lEFatmwZy5Ztu2/xox/9qHyR1fT8\nvS8gycYYFupOIiIiUt14M1uVmv4vcAbwlXA3aB/g81XWLV3cfwuYaWYPAf9Ckmb7ao8Fk5Ozm0Mx\ntS+5+yrgCuBB4OeEWhXhQv4DwDWhsOe61H7+BciZ2f1m9iDwue0alJzsZUtFOklSeyea2cNh+TvD\nci8C7wF+Gt7nLcCuoYvHOcBvQ6G211LvpXxkk48AB4YinQ+R1LzorZ3vMLMHQzeTPUlqjwDsR9LV\nRERERBrDncBSM1sUbricSlKIe1iYu7IzRURE6iX0Bc6FYpdLgOuAXdy9MELt+SxJ/+afD3D9Ce6+\nOTz/JvC4u184hE0s39+/AXe4+//Uax8iIiLNxMz82muvHbb9HXPMMbh7j4zLUMT8G0AG+K67f3G4\n2qPuJCIiIvU1AbjBzHIkXS7OHqkARvBNkoKkAwpiAO+3ZLjXFpIMke8MVcPKha4kh5OMaCIiIiIN\nwt3/F/jfkdi3MjFEREREREREmoSZ+XXXXTds+zv66KO3y8QYSaqJISIiIiIiIiJNQd1JRERERERE\nRJpIFI3dfISx+85FREREREREpKkoE0NERERERESkiZg1TImKYadMDBERERERERFpCgpiiIiIiIiI\niEhTUBBDRERERERERJqCamKIiIiIiIiINBHVxBARERERERERaXAKYoiIiIiIiIhIU1B3EhERERER\nEZEmou4kIiIiIiIiIiINTpkYIiIiIiIiIk1EmRgiIiIiIiIiIg1OmRgiIiIiIiIiTUSZGCIiIiIi\nIiIiDU6ZGCIiIiIiIiJNRJkYIiIiIiIiIiINTpkYIiIiIiIiIk1EmRgiIiIiIiIiIg1OmRgiIiIi\nIiIiTUSZGCIiIiIiIiIiDU5BDBERERERERFpCgpiiIiIiIiIiEhTUBBDRERERERERJqCCnuKiIiI\niIiINBEV9hQRERERERERaXDKxBARERERERFpIsrEEBERERERERFpcMrEEBEREREREWkiysQQERER\nEREREWlwysQQERERERERaSLKxBARERERERERaXAKYoiIiIiIiIhIU1B3EhEREREREZEmou4kIiIi\nIiIiIiINTpkYIiIiIiIiIk1EmRgiIiIiIiIiIg1OmRgiIiIiIiIiTUSZGCIiIiIiIiIiDU5BDBER\nERERERFpCgpiiIiIiIiIiEhTUE0MERERERERkSaimhgiIiIiIiIiIg1OmRgiIiIiIiIiTUSZGCIi\nIiIiIiIiDU5BDBERERERERFpCupOIiIiIiIiItJE1J1ERERERERERKTBKYghIiIiIiIi0kTMbNim\nAbTtPDN7xMweNLMvpeb/o5k9YWaPmtkxA33v6k4iIiIiIiIiIoNmZn8FnAjs4+55M5sZ5u8BnArs\nAcwDrjezXdw97u8+FMQQERERERERaSINXBPjbOCL7p4HcPf1Yf5JwE/D/JVm9iRwMHBbf3eg7iQi\nIiIiIiIiMhSWAm8ws9vMbIWZHRjmzwWeSy33HElGRr8pE0NERERERESkiYxkJoaZXQe0V3jpUyQx\nhh3c/VAzOwi4AtipyqZ8IPtXEENEREREREREALjjjju44447qr7u7kdXe83MzgauDMvdYWaxmc0A\nVgMLUovOD/P6zdwHFPwQERERERERkWFmZv7AAw8M2/723ntv3L2m1A8z+yAw190/Y2a7ANe7+46h\nsOdPSOpgzAOuB3b2AQQklIkhIiIiIiIiIkPhe8D3zOwBoAv4OwB3f9jMrgAeBgrAOQMJYIAyMURE\nRERERESaRiNnYgwHZWKIiIiIiIiINJEGHmK17jTEqoiIiIiIiIg0BQUxRERERERERKQpKIghIiIi\nIiIiIk1BNTFEREREREREmohqYoiIiIiIiIiINDhlYoiIiIiIiIg0EWViiIiIiIiIiIg0OGViiIiI\niIiIiDQRZWKIiIiIiIiIiDQ4ZWKIiIiIiIiINBFlYoiIiIiIiIiINDgFMURERERERESkKag7iYiI\niIiIiEgTUXcSEREREREREZEGp0wMERERERERkSaiTAwRERERERERkQanIIaIiIiIiIiINAUFMURE\nRERERESkKagmhoiIiIiIiEgTUU0MEREREREREZEGp0wMERERERERkSaiTAwRERERERERkQanIIaI\niIiIiIiINAUFMURERERERESkKSiIISIiIiIiIiJNQYU9RURERERERJqICnuKiIiIiIiIiDQ4ZWKI\niIiIiIiINBFlYoiIiIiIiIiINDhlYoiIiIiIiIg0EWViiIiIiIiIiIg0OAUxRERERERERKQpKIgh\nIiIiIiIiIk1BNTFEREREREREmohqYoiIiIiIiIiINDgFMURERERERESkKag7iYiIiIiIiEgTUXcS\nEREREREREZEGpyCGiIiIiIiIiDQFBTFEREREREREpCmoJoaIiIiIiIhIE1FNDBERERERERGRBqdM\nDBEREREREZEmokwMEREREREREZEGp0wMERERERERkSaiTAwRERERERERkQanIIaIiIiIiIiINAUF\nMURERERERESkKSiIISIiIiIiIiJNQUEMERERERERkSZiZsM29bNdf2NmD5lZ0cwOSM0/2szuNLP7\nw+NfpV47wMweMLMnzOzCvvahIIaIiIiIiIiIDIUHgLcBNwGemr8eOMHd9wHOAH6Yeu3bwHvdfSmw\n1MyO7W0HGmJVREREREREpIk06hCr7v4obN8+d7839ePDQJuZ5YAZwCR3/3N47b+AtwK/rbYPZWKI\niIiIiIiIyHA5GbjL3fPAPOC51Gurw7yqlIkhIiIiIiIi0kRGMhPDzK4D2iu89E/ufnUf6+4JXAAc\nPdD9K4ghIiIiIiIiIgDccsst3HLLLVVfd/cBBSDMbD5wJXC6uz8TZq8G5qcWmx/mVd+Ou/f2uoiI\niIiIiIg0CDPzNWvWDNv+5syZg7v3K/XDzG4E/sHd7wo/TwX+AHzG3X9VtuztwEeAPwPXABe5u2pi\niIiIiIiIiEj9mNnbzGwVcChwjZn9b3jpXGAJ8BkzuydMM8Jr5wCXAU8AT/YWwABlYoiIiIiIiIg0\nDTPztWvXDtv+2tvb+52JUU/KxBARERERERGRpqAghoiIiIiIiIg0BY1OIiIiIiIiItJERnKI1ZGm\nTAwRERERERERaQoKYoiIiIiIiIhIU1AQQ0RERERERESagmpiiIiIiIiIiDQR1cQQEREREREREWlw\nysQQERERERERaSLKxBARERERERERaXDKxBARERERERFpIsrEEBERERERERFpcApiiIiIiIiIiEhT\nUBBDRERERERERJqCghgiIiIiIiIi0hRU2FNERERERESkiaiwp4iIiIiIiIhIg1MmhoiIiIiIiEgT\nUSaGiIiIiIiIiEiDUxBDRERERERERJqCghgiIiIiIiIi0hRUE0NERERERESkiagmhoiIiIiIiIhI\ng1MmhoiIiIiIiEgTUSaGiIiIiIiIiEiDUxBDRERERERERJqCghgiIiIiIiIi0hQUxBARERERERGR\npqDCniIiIiIi8v+3d8csctVRGIffgyBZFxsrg2BlChsLt9BOSJXSFIIBU1koVoKN+C1sFEEtQsA2\nIBJFJDAGixQpXJXF2KloKTYLghyLHXBZkglm13gPPk8385+5d+ofe98FBjHsCQAAALBwIgYAAAAw\ngogBAAAAjGATAwAAAAaxiQEAAACwcCIGAAAAMIKIAQAAAIxgEwMAAAAGsYkBAAAAsHAiBgAAADCC\niAEAAACMIGIAAAAAIxj2BAAAgEEMewIAAAAsnIgBAAAAjCBiAAAAACPYxAAAAIBBbGIAAAAALJyI\nAQAAAIwgYgAAAAAjiBgAAADACIY9AQAAYBDDngAAAADHUFUvVNW3VfVnVT196P1TVfVRVX1dVd9V\n1ZuHznaqareqblXV23e7h4gBAAAAnITdJOeTrI68/2KSdPdTSXaSvFJVj6/P3k3ycncPwGBKAAAC\naklEQVSfSXKmqs5tuoGIAQAAABxbd+919/e3OfolyXZVPZBkO8kfSX6vqtNJHu7uG+vPXUry/KZ7\n2MQAAACAQaZtYnT3Z1V1MQcx46Ekr3f3b1X1RJKfDn305ySPbbqWiAEAAAAkSVarVVaro0+D/K2q\nPk/y6G2O3uruj+/wnZeSbCU5neSRJF9W1Rf38vuqu+/lewAAAMB9VlW9v79/3+63tbWV7v5Hf/pR\nVdeSvNHdN9ev30nyVXdfXr/+IMnVJNeTXOvuJ9fvX0jyXHe/eqdr28QAAAAATtrh8LGX5GySVNV2\nkmeT7HX3rznYxnimDp6RuZjkyqaLihgAAADAsVXV+ar6MQeR4pOquro+ei/Jg1W1m+RGkg+7+5v1\n2WtJ3k9yK8kP3f3pxnt4nAQAAABmmPA4yb/JsCcAAAAMMu2/k5wkj5MAAAAAI4gYAAAAwAgiBgAA\nADCCiAEAAACMYNgTAAAABjHsCQAAALBwIgYAAAAwgogBAAAAjGATAwAAAAaxiQEAAACwcCIGAAAA\nMIKIAQAAAIwgYgAAAAAjGPYEAACAQQx7AgAAACyciAEAAACMIGIAAAAAI9jEAAAAgEFsYgAAAAAs\nnIgBAAAAjCBiAAAAACPYxAAAAIBBbGIAAAAALJyIAQAAAIwgYgAAAAAjiBgAAADACIY9AQAAYBDD\nngAAAAALJ2IAAAAAI4gYAAAAwAg2MQAAAGAQmxgAAAAACydiAAAAACOIGAAAAMAINjEAAABgkP/z\nJkZ193/9GwAAAADuyuMkAAAAwAgiBgAAADCCiAEAAACMIGIAAAAAI4gYAAAAwAh/AQgaAJlwbvDi\nAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "## Create a 2D surface plot\n", + "\n", + "# Create matrices of the positions and data values.\n", + "points = sensor[['latitude', 'longitude']].as_matrix()\n", + "values = sensor[sensor_data]\n", + "\n", + "# Create a grid over the range of the data.\n", + "grid_x, grid_y = np.mgrid[sensor.latitude.max():sensor.latitude.min():1000j,\n", + " sensor.longitude.min():sensor.longitude.max():1000j]\n", + "grid_z = scipy.interpolate.griddata(points, values, (grid_x, grid_y), method='cubic')\n", + "\n", + "# Plot the sensor data as an image.\n", + "fig = plt.figure(figsize=(20,10))\n", + "plt.imshow(grid_z, aspect='equal', cmap='Greys',\n", + " extent=[sensor.longitude.min(), sensor.longitude.max(),\n", + " sensor.latitude.min(), sensor.latitude.max()])\n", + "\n", + "plt.colorbar()\n", + "plt.title(\"Sensor: \" + sensor_name + \"\\n\")\n", + "plt.xlabel('Longitude (degrees)')\n", + "plt.ylabel('Latitude (degrees)')\n", + "plt.hold(True)\n", + "\n", + "# Plot the trajectory of the vehicle as an overlay.\n", + "plt.plot(points[:,1], points[:,0], 'c-', linewidth=3)\n", + "plt.plot(points[0,1], points[0,0], 'co')\n", + "plt.show()\n", + "\n", + "# Save the figure to disk.\n", + "fig.savefig(sensor_name + '.png', dpi=fig.dpi)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "## Create a 3D trail plot\n", + "x = sensor['longitude'].as_matrix()\n", + "y = sensor['latitude'].as_matrix()\n", + "z = sensor[sensor_data].as_matrix()\n", + "\n", + "import matplotlib.cm as cm\n", + "import matplotlib.colors as colors\n", + "\n", + "# Create normalized color entries for each sensor reading.\n", + "norm = colors.Normalize(min(z), max(z))\n", + "colors = cm.jet(norm(z))\n", + "m = cm.ScalarMappable(cmap=cm.jet, norm=norm)\n", + "m.set_array(colors)\n", + "\n", + "# Create a 3D plot that creates a vertical trail.\n", + "from mpl_toolkits.mplot3d import Axes3D\n", + "fig = plt.figure(figsize=(20,10))\n", + "ax = fig.add_subplot(111, projection='3d')\n", + "ax.bar3d(x, y, [min(z)] * len(z), 5e-5, 5e-5, z - min(z), color=colors, alpha=0.8, edgecolor='none')\n", + "\n", + "plt.xlabel('Longitude (degrees)')\n", + "plt.ylabel('Latitude (degrees)')\n", + "plt.title('Sensor: ' + sensor_name, fontsize=20)\n", + "\n", + "fig.colorbar(m, shrink=0.5, aspect=5)\n", + "fig.show()\n", + "\n", + "# Save the figure to disk.\n", + "fig.savefig(sensor_name + '_3d.png', dpi=fig.dpi)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.9" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From 4cf5452a80311d5fd9c44e069b0f58135240839c Mon Sep 17 00:00:00 2001 From: Pras Velagapudi Date: Sun, 27 Nov 2016 12:45:26 -0500 Subject: [PATCH 03/43] Renamed "temp" to "temperature". This makes it consistent everywhere. --- src/platypus/io/logs.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/platypus/io/logs.py b/src/platypus/io/logs.py index 371ecc1..64780da 100644 --- a/src/platypus/io/logs.py +++ b/src/platypus/io/logs.py @@ -57,10 +57,10 @@ """ _REGEX_ES2_V4_0_0 = re.compile( - r"^ES2: \[e, (?P[\d\.]+), (?P[\d\.]+)\]") + r"^ES2: \[e, (?P[\d\.]+), (?P[\d\.]+)\]") """ Defines a regular expression that represents a pose record of the form: -'ES2: [e, , ]' +'ES2: [e, , ]' This format is used in v4.0.0 vehicle log entries. """ @@ -76,7 +76,7 @@ _DATA_FIELDS_v4_1_0 = { 'BATTERY': ('voltage', 'm0_current', 'm1_current'), - 'ES2': ('ec', 'temp'), + 'ES2': ('ec', 'temperature'), 'ATLAS_DO': ('do',), 'ATLAS_PH': ('ph',), } @@ -86,7 +86,7 @@ _DATA_FIELDS_v4_2_0 = { 'BATTERY': ('voltage', 'm0_current', 'm1_current'), - 'ES2': ('ec', 'temp'), + 'ES2': ('ec', 'temperature'), 'ATLAS_DO': ('do',), 'ATLAS_PH': ('ph',), } @@ -309,7 +309,7 @@ def read_v4_0_0(logfile, filename): data_sensors['es2'] = [] data_sensors['es2'].append([timestamp, float(m_es2.group('ec')), - float(m_es2.group('temp'))]) + float(m_es2.group('temperature'))]) continue m_sensor = _REGEX_SENSOR_V4_0_0.match(message) From 8181975712c6805a1f290b647b8f464296dedb9a Mon Sep 17 00:00:00 2001 From: Pras Velagapudi Date: Sun, 27 Nov 2016 12:47:14 -0500 Subject: [PATCH 04/43] Added gitignore for IPython checkpoints. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index ba74660..8f67b15 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,6 @@ docs/_build/ # PyBuilder target/ + +# IPython +.ipynb_checkpoints From 2eaa59ab144f553c25532930876258d8445fc98a Mon Sep 17 00:00:00 2001 From: Pras Velagapudi Date: Sun, 27 Nov 2016 13:07:03 -0500 Subject: [PATCH 05/43] Cleaned up and nominally working data interpolation script. --- notebooks/Data_Interpolation.ipynb | 78 ++++++++---------------------- 1 file changed, 20 insertions(+), 58 deletions(-) diff --git a/notebooks/Data_Interpolation.ipynb b/notebooks/Data_Interpolation.ipynb index 4e0c88d..c5d1782 100644 --- a/notebooks/Data_Interpolation.ipynb +++ b/notebooks/Data_Interpolation.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": { "collapsed": false }, @@ -34,32 +34,11 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Available sensors/channels:\n", - " ES2, ec\n", - " ES2, temp\n" - ] - }, - { - "ename": "ValueError", - "evalue": "Unknown format code 's' for object of type 'long'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtypes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0;32mprint\u001b[0m \u001b[0;34m\" {:s}, {:s}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;31mValueError\u001b[0m: Unknown format code 's' for object of type 'long'" - ] - } - ], + "outputs": [], "source": [ "# Print the available sensors and channels for this logfile.\n", "print \"Available sensors/channels:\"\n", @@ -67,7 +46,7 @@ " if s == 'pose' or s == 'BATTERY':\n", " continue\n", " for c in data[s].dtypes.keys():\n", - " print \" {:s}, {:s}\".format(s, c)" + " print \" {:s}, {:s}\".format(s, str(c))" ] }, { @@ -85,58 +64,39 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'sensor_name' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m# Extract the pose and the sensor data of interest.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mpose\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'pose'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0msensor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msensor_name\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;31m# Linearly interpolate the position of the sensor at every sample.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mNameError\u001b[0m: name 'sensor_name' is not defined" - ] - } - ], + "outputs": [], "source": [ "# Extract the pose and the sensor data of interest.\n", "pose = data['pose']\n", + "pose_times = pose.index.values.astype(np.float64)\n", + "\n", "sensor = data[sensor_name]\n", + "sensor_times = sensor.index.values.astype(np.float64)\n", "\n", "# Linearly interpolate the position of the sensor at every sample.\n", - "sensor_pose = scipy.interpolate.interp1d(pose['time'], pose[['latitude', 'longitude']],\n", - " axis=0, bounds_error=False)\n", + "sensor_pose_interpolator = scipy.interpolate.interp1d(pose_times, pose[['latitude', 'longitude']],\n", + " axis=0, bounds_error=False)\n", "\n", "# Add the position information back to the sensor data.\n", - "sensor = sensor.join(pandas.DataFrame(pose_interp(sensor['time']), sensor.index,\n", + "sensor = sensor.join(pandas.DataFrame(sensor_pose_interpolator(sensor_times), sensor.index,\n", " columns=('latitude', 'longitude')))\n", "\n", "# Remove columns that have NaN values (no pose information).\n", - "sensor = sensor[np.isfinite(sensor['time'])]" + "sensor_valid = np.all(np.isfinite(sensor), axis=1)\n", + "sensor = sensor[sensor_valid]" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABDEAAAI8CAYAAADoRDBpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmcZHV56P/PU9XdM8MwMGwyzDBsgiiLuCAoGh0TRWJu\nJLkxuMQrYBIlXhO9rkiSK9lw+WlUYjD+rkb9JQFRb2IgccPEdokLriyyo8MywLDDzDB0d1U9vz/O\nqZkzRS/VPb1UTX/er9d59alTZ6vqqlPnPOf5Pt/ITCRJkiRJknpdbaF3QJIkSZIkqRsGMSRJkiRJ\nUl8wiCFJkiRJkvqCQQxJkiRJktQXDGJIkiRJkqS+YBBDkiRJkiT1hYGF3gFJkiRJktSdiMj53mZm\nxnxvcyJmYkiSJEmSpL5gJoYkSZIkSX0kYv4SIzLnPfFjUmZiSJIkSZKkvmAQQ5IkSZIk9QWbk0iS\nJEmS1EdsTiJJkiRJktTjzMSQJEmSJKmPzGcmRq8xE0OSJEmSJPUFMzEkSZIkSeojtdr85SM0m815\n21Y3zMSQJEmSJEl9wUwMSZIkSZL6iDUxJEmSJEmSepyZGJIkSZIk9REzMSRJkiRJknqcQQxJkiRJ\nktQXbE4iSZIkSVIfsTmJJEmSJElSjzMTQ5IkSZKkPmImhiRJkiRJUo8zE0OSJEmSpD5iJoYkSZIk\nSVKPMxNDkiRJkqQ+Uqst3nyExfvKJUmSJElSXzETQ5IkSZKkPmJNDEmSJEmSpB5nJoYkSZIkSX3E\nTAxJkiRJkqQeZxBDkiRJkiT1BZuTSJIkSZLUR2xOIkmSJEmS1OPMxJAkSZIkqY+YiSFJkiRJktTj\nzMSQJEmSJKmPmIkhSZIkSZLU48zEkCRJkiSpj9RqizcfYfG+ckmSJEmS1FfMxJAkSZIkqY9YE0OS\nJEmSJGkKEbE0Ir4fET+NiGsi4t3l9L0j4rKIuCEivhoRKyvLvDMiboyI6yLi5J3ZvkEMSZIkSZLU\nlcx8FHh+Zj4FeDLw/Ih4DnA2cFlmPgH4j/IxEXEU8DLgKOAU4IKImHEswiCGJEmSJEl9JCLmbRhP\nZj5Sjg4BdeAB4CXAp8vpnwZ+oxw/FbgoM8cycz1wE3DCTF+7QQxJkiRJktS1iKhFxE+BjcDXM/Nn\nwP6ZubGcZSOwfzm+Gri9svjtwJqZbtvCnpIkSZIk9ZGFLuyZmS3gKRGxJ/CViHh+x/MZETnZKma6\nbYMYkiRJkiQJgNHRUUZHR7uaNzMfioh/B54ObIyIVZl5V0QcANxdzrYBWFtZ7MBy2ozYnESSJEmS\npD4ylzUwlixZwooVK7YN42x733bPIxGxDHgh8BPgEuD0crbTgS+U45cAL4+IoYg4FDgCuHymr91M\nDEmSJEmS1K0DgE+XPYzUgH/IzP+IiJ8An42I3wXWA6cBZOY1EfFZ4BqgAbw+M2fcnCR2YllJkiRJ\nkjSPIiJXrVo1b9u76667yMyFLcJRYXMSSZIkSZLUF2xOIkmSJElSH6nVFm8+wuJ95ZIkSZIkqa+Y\niSFJkiRJUh+J6JkSFfPOTAxJkiRJktQXDGJIkiRJkqS+YHMSSZIkSZL6iM1JJEmSJEmSepyZGJIk\nSZIk9REzMSRJkiRJknqcmRiSJEmSJPURMzEkSZIkSZJ6nJkYkiRJkiT1ETMxJEmSJEmSepyZGJIk\nSZIk9ZFabfHmIyzeVy5JkiRJkvqKQQxJkiRJktQXbE4iSZIkSVIfsbCnJEmSJElSjzMTQ5IkSZKk\nPmImhiRJkiRJUo8zE0OSJEmSpD5iJoYkSZIkSVKPMxNDkiRJkqQ+YiaGJEmSJElSjzMTQ5IkSZKk\nPlKrLd58hMX7yiVJkiRJUl8xE0OSJEmSpD5iTQxJkiRJkqQeZxBDkiRJkiT1BZuTSJIkSZLUR2xO\nIkmSJEmS1OPMxJAkSZIkqY/YxaokSZIkSVKPMxNDkiRJkqQ+Yk0MSZIkSZKkHmcmhiRJkiRJfcSa\nGJIkSZIkST3OTAxJkiRJkvqINTEkSZIkSZJ6nJkYkiRJkiT1ETMxJEmSJEmSepxBDEmSJEmS1Bds\nTiJJkiRJUh+xi1VJkiRJkqQeZyaGJEmSJEl9xMKekiRJkiRJPc5MDEmSJEmS+og1MSRJkiRJknqc\nmRiSJEmSJPURa2JIkiRJkiT1OIMYkiRJkiT1kVqtNm9Dp4hYGxFfj4ifRcTVEfFHHc+/JSJaEbF3\nZdo7I+LGiLguIk7emdducxJJkiRJktStMeB/ZeZPI2J34EcRcVlmXhsRa4EXAre0Z46Io4CXAUcB\na4CvRcQTMrM1k42biSFJkiRJkrqSmXdl5k/L8c3AtcDq8um/Bt7escipwEWZOZaZ64GbgBNmun0z\nMSRJkiRJ6iO9UtgzIg4Bngp8PyJOBW7PzCs79m818L3K49spMjJmxCCGJEmSJEkC4KGHHuLhhx+e\ncr6yKcnngTcCLeAciqYk22aZZPGc6f4ZxJAkSZIkqY/MZSbGypUrWbly5bbHt99++3jbHwT+L/CP\nmfmFiDgWOAS4oty3AylqZZwIbADWVhY/sJw2I9bEkCRJkiRJXYkiSvEJ4JrM/BBAZl6Vmftn5qGZ\neShFk5GnZeZG4BLg5RExFBGHAkcAl890+2ZiSJIkSZLUR8br+nQePRt4FXBlRPyknHZOZn6pMs+2\n5iKZeU1EfBa4BmgAr89Mm5NIkiRJkqS5lZnfZopWHZl5WMfj84DzZmP7BjEkSZIkSeojvdI7yUKw\nJoYkSZIkSeoLZmJIkiRJktRHFrgmxoJavK9ckiRJkiT1FTMxJEmSJEnqI9bEkCRJkiRJ6nEGMSRJ\nkiRJUl+wOYkkSZIkSX3E5iSSJEmSJEk9zkwMSZIkSZL6iF2sSpIkSZIk9TgzMSRJkiRJ6iPWxJAk\nSZIkSepxZmJIkiRJktRHrIkhSZIkSZLU48zEkCRJkiSpj1gTQ5IkSZIkqccZxJAkSZIkSX3B5iSS\nJEmSJPURm5NIkiRJkiT1ODMxJEmSJEnqI3axKkmSJEmS1OPMxJAkSZIkqY9YE0OSJEmSJKnHmYkh\nSZIkSVIfsSaGJEmSJElSjzMTQ5IkSZKkPmJNDEmSJEmSpB5nJoYkSZIkSX3EmhiSJEmSJEk9ziCG\nJEmSJEnqCzYnkSRJkiSpj1jYU5IkSZIkqceZiSFJkiRJUh8xE0OSJEmSJKnHmYkhSZIkSVIfMRND\nkiRJkiSpx5mJIUmSJElSHzETQ5IkSZIkqceZiSFJkiRJUh8xE0OSJEmSJKnHGcSQJEmSJEl9weYk\nkiRJkiT1EZuTSJIkSZIk9TgzMSRJkiRJ6iNmYkiSJEmSJPU4MzEkSZIkSeojZmJIkiRJkiT1ODMx\nJEmSJEnqI7Xa4s1HWLyvXJIkSZIk9RWDGJIkSZIk9ZGImLdhnG3/fURsjIirKtNOiIjLI+InEfGD\niHhG5bl3RsSNEXFdRJy8s6/dIIYkSZIkSerWJ4FTOqa9D/jTzHwq8L/Lx0TEUcDLgKPKZS6IiJ2K\nQ1gTQ5IkSZKkPrKQvZNk5rci4pCOyXcCe5bjK4EN5fipwEWZOQasj4ibgBOA7810+wYxJEmSJEnS\nzjgb+HZEvJ+ixcezyumr2TFgcTuwZmc2ZHMSSZIkSZK0Mz4B/FFmHgT8L+DvJ5k3d2ZDZmJIkiRJ\nktRH5rI5yV133cXGjRunu9gJmfmCcvzzwMfL8Q3A2sp8B7K9qcmMGMSQJEmSJEkArFq1ilWrVm17\nfOWVV3az2E0R8bzM/Abwy8AN5fRLgAsj4q8pmpEcAVy+M/tnEEOSJEmSpD6ykIU9I+Ii4HnAvhFx\nG0VvJK8F/jYilgBby8dk5jUR8VngGqABvD4zbU4iSZIkSZLmXma+YoKnTpxg/vOA82Zr+wYxJEmS\nJEnqIwuZibHQ7J1EkiRJkiT1BTMxJEmSJEnqI2ZiSJIkSZIk9TgzMSRJkiRJ6iNmYkiSJEmSJPU4\nMzEkSZIkSeojZmJIkiRJkiT1OIMYkiRJkiSpL9icRJIkSZKkPmJzEkmSJEmSpB5nJoYkSZIkSX3E\nTAxJkiRJkqQeZyaGJEmSJEl9xEwMSZIkSZKkHmcmhiRJkiRJfcRMDEmSJEmSpB5nJoYkSZIkSX3E\nTAxJkiRJkqQeZxBDkiRJkiT1BZuTSJIkSZLUR2xOIkmSJEmS1OPMxJAkSZIkqY+YiSFJkiRJktTj\nzMSQJEmSJKmPmIkhSZIkSZLU48zEkCRJkiSpj5iJIUmSJEmS1OPMxJAkSZIkqY+YiSFJkiRJktTj\nzMSQJEmSJKmPmIkhSZIkSZLU4wxiSJIkSZKkvmBzEkmSJEmS+ojNSSRJkiRJknqcmRiSJEmSJPUR\nMzEkSZIkSZJ6nJkYkiRJkiT1ETMxJEmSJEmSepyZGJIkSZIk9REzMSRJkiRJknqcmRiSJEmSJPUR\nMzEkSZIkSZJ6nEEMSZIkSZLUF2xOIkmSJElSH7E5iSRJkiRJUo8zE0OSJEmSpD5Sqy3efITF+8ol\nSZIkSVJfMYghSRIQEc+JiO9ExIMRcV9EfDsijl/o/ZquiNg9IjZHxBcr034WEZvKoRERWyuP3xkR\nZ0TEt6ZY7xkR0YqI08Z57pyI+Hm5vtsi4jNd7OdwuR8PR8RDEfHDiHhHRAx1zHdURFxS/l8ejoj/\njIhnTec9kSRpVxMR8zb0GoMYkqRFLyL2AP4N+DCwF7AG+DNgZCH3qy1KXc7+W8CtwLqI2B8gM4/O\nzBWZuQL4FvA/248z891drvd04Crg1R37djrwKuBXyvUfD3yti/VluR97AKuAtwAvB6rBl8cD/wVc\nARwCHAD8C/DViHhml/stSZJ2IQYxJEmCJwCZmRdn4dHMvCwzr2rPEBGviYhrIuL+iPhyRBxUea4V\nEa+LiBsi4oGI+EjlucMj4htlJsE91SyFiDgpIn5QPnd5NcOgzFT4y4j4L2ALcGiXr+V04OMUF/+v\nmmCead1WiYiDgWcDZwIvbAdHSscDX8nMXwBk5sbM/Hi3qy6X2ZqZ3wBeAjwrIn6tfP5c4L8y808z\n88HM3JKZfwP8A/De6bwGSZJ2JWZiSJK0uF0PNCPiUxFxSkTsVX0yIk4F3gn8JrAvRTbDRR3r+DWK\nC/onA6dFxMnl9L8AvpyZKykyPM4v17k38O/Ah4C9gb8G/r1j268Cfg/YHbg1Ii6NiLdP9CLKYMNz\ngc+Ww6snmDUnWscEXg18IzN/DPwQ+J3Kc98DXh0Rb42I4yOiPo317rAfmXlbuf7nlJNeCHxunOU+\nBzw7IpZMY1uSJGkWRMTfR8TGiKje7Pl/IuLaiLgiIv45IvasPPfOiLgxIq6rnB/NmEEMSdKil5mb\nKC6cE/g/wN0R8a8R8bhylrOAd2fm9ZnZAt4NPCUi1lZW857MfLi8EP868JRy+ihwSESsyczRzPxO\nOf3XgOsz858ys5WZnwGuo8hGoNyXT2XmteXzjcz89cx83yQv5X8Al2fm7cA/A0dFxFMmmb9br2Z7\nMOFzVIIjmflPwB8CLwKGgY2TBVq6cAdFUAdgH+DOcea5k+IcZu9xnpMkaZe3wJkYnwRO6Zj2VeDo\nzDwOuIHi5g8RcRTwMuCocpkLImKn4hAGMSRJAjLzusw8MzPXAscAqymyJAAOBj5cNhV5ALivnL6m\nsoq7KuOPACvK8bdTNJu4PCKujogzy+mrKWpXVN1STm+7bZovY1uwITPvowgqnD7NdewgIp5NUY/i\nn8tJnweOjYjj2vNk5oWZ+UJgT4qAz1/sxJ2WA4H7y/F72fH9aDsAaAEPzHAbkiRphjLzW3T8BpfN\ncFvlw+9T/J4DnApclJljmbkeuAk4YWe2bxBDkqQOmXk98GmKYAYUwYbXZuZelWF5Zn6vi3VtzMzX\nZuYa4HUUdyAeD2ygCI5UHVxO37Z4t/scEScBhwN/EhF3RsSdwLOAV06ziUen0ymCMFeV6/xBZfoO\nMrOZmZ8HrgSOnu6GysyWp1E014GiQOhvjzPracB3MvPR6W5DkqRdQY/XxHgN2wt1rwZurzx3Ozve\nBJq2gZ1ZWJKkXUFEHEnRvOPizNxQXky/AvhuOcvfUWQXXJGZ15TtPE/OzPHqNUClcGZE/Dbw3bKJ\nx4MUgYkm8CXgbyLiFRTZE78FPJGil5THrKcLp1OkclbrYOxGEVD41S7WG2WNic7nTgN+n6J+R9tL\ngf8dEW+jaMJyN0XgYQtFs5KjKe7CTCXKDe8GPAP4IPD9zGyf+PwZ8IOI+EvgA0ADOKPc5gu7WL8k\nSZqmW265hVtv7UwW7U5E/DEwmpkXTjLbdGtz7cAghiRJsAk4EXhzRKykCDZcCrwNIDO/EBG7A58p\ni2c+RBEwaAcxOn+MszLteOCDZeBjI/BHZTolEfHfKLp1/ShwI/DfMvP+jvVsExFfBL6Zme/pmL6U\nImPhf2Tm3R3P/QNFYKMaxBhvf08CtnZMezVFYOL/y8xmZZ2fBP6cImDxEHAORVvXOrAeOKtS+2My\nH4mID5bjN1G8nx/YtgOZN0XEc4D3lOutUWSCnJyZ30WSJM26gw8+mIMP3p4s+u1vf7ur5SLiDODF\nwK9UJm8AqjXEDmTHrNNpi8ydCoJIkiRJkqR5EhF5zjnnzNv2zjvvPDJzh0zNiDgEuDQzjy0fn0Jx\nI+J5mXlvZb6jgAsp6mCsoWgqenjuRCDCTAxJkiRJktSViLgIeB6wb0TcBryLojeSIeCyso7GdzPz\n9WUz3M8C11A0C339zgQwwCCGJEmaIxGxmfHbvZ6Smf813/sjSdKuYoYFN2dFZr5inMl/P8n85wHn\nzdb2DWJIkqQ5kZm7L/Q+SJKkXYtBDEmSJEmS+shCZmIstNpC74AkSZIkSVI3zMSQJEmSJKmPmIkh\nSZIkSZLU4wxijCMi3hIRrYjYe4LnT4mI6yLixoh4R2X6X0TEFRHx04j4j4hY27HcQRGxOSLeUpn2\n5XL+n0XEJyJisJx+VkRcGRE/iYjvRsRxXez330fExoi4auavXpIkSZLUyyJi3oZes2iDGBGxLiI+\nOc70tcALgVsmWK4OfAQ4BTgKeEVEPKl8+n2ZeVxmPgX4AkV/uVV/Dfx7x7SXZuZTMvNoYE/gZeX0\nf8rMJ2fmUym6o/lAFy/rk+V+SZIkSZK0y1m0QQzG77ceikDD2ydZ7gTgpsxcn5ljwGeAUwEyc1Nl\nvt2Be9sPIuI3gJ8D1+ywE5mby+cHgaH2MlOs620RcXmZ9XFuZV3fAh6YZN8lSZIkSepbUxb2jIiV\nwLOAQygu/NcD383Mh+Z0z+beY/JiIuJU4PbMvHKStJk1wG2Vx7cDJ1bW8VfA/wAeAZ5ZTtudIjDy\nAuBt42z3K8AzgMsy88uV6a8H3gwsB04qp50MHJ6ZJ0REDfjXiPilMoAhSZIkSdrF9WIzj/kyYSZG\nRPxSRFwCfBN4OXAQRSDjFcC3IuKSiHjOvOzlLIqI70XET4D/A7ykrDnxk4h4CfBOdmwCMt4nY6IM\njuLJzD/OzIOATwEfLCefC3wwMx8Zb52Z+SLgAGBJRJxemX5BZh5OEcj4+3LyycDJ5Wv4EXAkcPjk\nr1qSJEmSpP43WSbGbwJvycwbx3syIp4AnAV8ey52bK5kZjs74nnAGZl5Zvn4GOBQ4IoyqnUg8KOI\nOCEz766sYgNQLdi5liIbo9OFwBfL8ROA34qI9wErgVZEbM3MCyr7NRIR/5ciq+PTHeu6GPi7yuN3\nZ+b/O42XLUmSJEnaRdRqi7cyxISvPDPfPFEAo3z+hsx882Qrn6gXj455zi+fvyIinjrVshGxd0Rc\nFhE3RMRXy+YuRMQhEbG1kllxwXjbq2664/VcnZn7Z+ahmXkoRWDiaR0BDIAfAkeU2xuiKMR5SbkP\nR1TmOxX4Sbnu51bW+yHgrzLzgohYHhEHlMsOAP+tvUzHun4NuLIc/wrwmohYXs63JiL2m+K1SpIk\nSZLU96YM30TEmyJizyh8ogwQvKiL5SbrxaM9z4sp6jscAbwW+GgXy55NUTviCcB/lI/bbsrMp5bD\n66fYxWTypiHbnouI1RHx7wCZ2QDeQBFMuAa4ODOvLWd9d0RcFRE/BdYBb2Fyu1PUtLgC+DFwK9ub\njfzPiLi6bDbyh8CZ5fYvo8jy+G5EXAl8rlwPEXER8B3gCRFxW0ScOcX2JUmSJEl9ZjF3sTplYU/g\nNZn5oTJwsTdF0cp/oLiIn8y2XjwAIqLdi8e1lXleQtl0IjO/HxErI2IVRbOOiZZ9CfC8cvlPA8Ps\nGMjoSmZ+A/jGJM8fVhm/gyIbov34S8CXxlnmpV1s988q4xsp3qfx5nvTJOs4Hzh/nOmvmGr7kiRJ\nkiT1q24a0rRDL78G/ENmXt3lusfrxWNNl/OsnmTZ/cuLf4CNwP6V+Q4tM0WG+7HoqCRJkiRJUzET\nY3I/ioivAocBZ0fEHkCri+Um7cWjopt3JcZbX2ZmRLSn3wGszcwHIuJpwBci4ujM3NTlfkiSJEmS\npB7WTRDjd4HjgJ9n5iMRsQ9lfYYpdNOLR+c8B5bzDI4zfUM5vjEiVmXmXWVRzLsBMnMUGC3HfxwR\nNwNHUNSa2KYS9JAkSZIk7YIys/dSCGZRL2ZIzJdughgJHE3Rc8afA8uBpV0st60XD4osiZcBnTUb\nLqEokvmZiHgm8GBmboyI+yZZ9hLgdOC95d8vAETEvsADmdmMiMMoAhg/H/cF7eIfaM2viDg3M89d\n6P3oZTE8HMBewCpgH4quhleW0/YBHlcZ9idzf4qsrynt02rxic2bu0rp6hcXXXQRr3jF9ErcLOYf\nMk3uwgsv5Hd+53cWeje0C7nwwgt55StfudC7sUvI9N4aTPyZ6ub9mez37+p6nXN2223qdWRmNho/\nZ3DwRmCkHEbLv1srw73ATcDNwPpct250ypVrQXjjetfWTRDjAqAJ/ApFEGNzOe34yRbKzEZEtHvx\nqAOfyMxrI+J15fMfy8wvRsSLI+ImYAvbe+AYd9ly1e8BPhsRvwusB04rpz8X+POIGKNo7vK6zHyw\nmzdB0szE8HCNIjBxQDlUx7cPrdZqarVujjfliru/IL+vVuNTS5bw3LExDm61ujqo7YoW+kTYIEpv\nW+jPx3xYjJ/Bhfq/Zuai+Ez1u377H7X3t/pd3tnv9eUD3Z0VZEQwOPh44PHTWH0rhodvZXtQo/33\nZooM8vtz3br++ieoryzG3722br7ZJ2bmU8uuPsnM+yNisJuVj9eLR2Z+rOPxG7pdtr194AXjTP9n\n4J+72S9J3SuzKNYAxwDHDN5zz3MOWr78BVuXLl0+EEGjmwNorZsawjuqZ7Iykz3Lv3u0Wtse753J\nRUuWcFe53i8sWcIXlixhaSaHNJvs32qxTybLMlkKLM1kaSZLKKKi215bx/bqFAfFeiYDwLJMds9k\nt47l9FgLebK8mH/EtV2/XbCpd/jZmRvTPTbPxbH88GZz2/gerRYf37KFByJYX6/zi1qN9bUav6jX\n2TiD8xSKDhIOKYfHXJsAozE8fBdw5zjDeuA6YIOBDmn6uglijEbEtvP3iNiP7gp7SovF8ELvwEys\nXLnycaeffvp7XvCCF5wJRbrVPRHcUatxR63GLbUat9Tr7Far8UjlxGJsv/24eQbbW1YGH/ZotVgO\nLM9keRkkWJnJnq0We1aCFrtnTtp9Ui2TD3SkiD4awXUDA1w3g/3rZv93z2SPch/3KPd32+Pyb/t1\nLKO7qsXjOeaYY2Zz13d5C3UB0i/Bk2OPPXahd0F9rvM7dswxx3jhvwjM5zFuro5TJzYaLM9kSwQP\n12p8Z2CAX240OKDR4FmV+bYCt9dqPFjenBkDGsBoBKOVv/dHcGetxp21GvdFkJO/R0PAQeUwkXti\nePgLwOeA/8h167zGkroQU/0IRcSrKJpsPB34NPBS4E8y87Nzv3uzLyLSmhja1QwNDS359V//9T86\n44wz3jfe89Xv+QjFD/VttRq31evcVqtxe63Gxlqtu6yKDisy2bvVYu9M9spkr+p45XE3hXTG29/J\n3FKrccXAANfX61xfr3PPzO6kzImBMkgzCAxlMsT2bpaSIhLc/rsEiuBNq8VeZRCnOl6jCDI9GsGm\nyvBwx9+EHbY31LH99t9xp43zOICRiKJxcMSO4xTZKUvLgM3uZZBnReV1d0qKE8XNEduGhyuv59EI\nmuVrbVbGG0Argkb5uAU0yufrwG6Z7FZue58yC2ffVov9yuBS73wqJGn+9Eugda59ZmiIC5csAWBN\nq8UFW7bMSh2tUWBjrcZd5Q2gu8rgxsYI7q/V2Dr99/9y4Ddz3bo7ZmH3Fr1d/ZovIvK8886bt+2d\nc845PVVXcspMjMz8x4j4EUVNDIBTK/UpJM2B5z73uS8/66yzPrJixYp9xnu+8yL/85///GOmPwLc\nVq9vD1iUQYu7p75z8BjLMzm42eSgVos1rRarWi32z+RxrRZTl8uavuqJ12QBjYNbLQ4e3V5T657y\nDsndtRoPlBfFj1Jc/LfH22urvgcJO1wwN8u7MFsj2FIO09WI4KFFfAI5WDbPqVM0z0lgSwSteX5P\nBsuAxj7tpkHtgeJzvVtlWA47PF7KzLNppmMMuKlMZ74vggci2Fp+BsciWFoGhlZk7hCg2bfVYsk8\n7J+khWMgYuf9+ugo/zI0xNYINtRqXFer8aTWzic8DAFrW62iO8VKs5W2RykyNx4oz0nuL4/v95fB\njttqtc7zixOe2GxuOOmFL3zVdy677J92egelXdiUmRgAEfFLwOGZ+cmyOcnumfmLOd+7ObCrR+XU\nW9auXfuks84662+f/OQnP7+b+WeSnrsJtmVUVLMr7ptBVsLeZYBidavF2larCBKUmRSdX5rqvs71\nSdZCpy032Z5B8FCtxsNlgKL9d9t45bkRTzz7Xq0MduxW1lQZZHvGymAZ9Nij1WJFmfEx3t/xggyb\ngZvrdX42MMDPyiyisRl+XvZotdg3k/1aLfZttViVyQHtQGOrxdCMX72knWUAonf8zZIlXDZUHBFP\nGR3l9SMjC7xHRVbhdfU63xoY4IuDg9turvzG6CivKffv3nvvve3MM8+crDmKJrCrX/NFRL773e+e\nt+29851AV77NAAAgAElEQVTv7K9MjIg4l6IpyZHAJykCj/8IPHtO90zqMXvuued+r371q887+eST\nf6/bZS644IJZq+L+KHBLvc4ttRrr63VuLYMWD04zWFHL5IBM1jabxR2EcljdarGsy3V0vp7xKorP\npohY0EBGHdidosnEqnHutoynnQFSbUsLxZ39oKgG1h7fCjxYtsXdNlQeJ8XBepCi+U77wrk67FFm\nPrS3NVYdL5t/jHXsT+c8ox2Pk6J5yVJgSXlBviRz23iTIltlawRbgE2VZiITZVy0swrazU+qF/1L\nc8fiqrX2OMXndtt4ZZ5GBI9QZHk8HMF9tRr3R3BvrcY9tRqbd+Iz2YpgM+zUOoYqr3FrGeCaQYrx\nhB6u1XgY+Hn9saVnI5N9y6DG6lZr23f+wAkCk5Iey0DEruH5jca2IMaPBgbIkZEFPwbWgKOaTY5q\nNtk3k0+XTV6+MDTEUc0mxzUaXLlq1dozvvKVfEajwUGtFueee+6v/uhHP/rywu65tPC6qYlxBfBU\n4EeZ+dRy2pWZ+eR52L9Zt6tH5dS9l73sZX/8yle+8s9qtdpOdTwx2xfXTeDOWo1by2F9vc76sq3l\ndJqBDGRuy6hY22pxUCVY0VX3QhPo4pixE2vfuW2rN7QomuW0a1g0IwiKjIad+ezNxCPAvWVgY0sE\nj1SGLbDD4/a0djOi0Xm8eFndbHJoq8XeZdOXdl2RgUwerdQPubdW497y730RNGe4j8szObASyNyn\nIzC2vAxe2SuP+p1BCEHxe/Q7u+++LYj8d5s3s7qHzilawF8uW8YPJ+gSdiCTv9i6laMrN1JuueWW\nq9/whjdYuXkCu/o1X0Tke97znnnb3tlnn91fmRjASGa22j8CEbG825VHxCnAhyjOgz6eme8dZ57z\ngV+lONc8IzN/MtmyEbE3cDFwMEX3RKdl5oOV9R0EXAO8KzM/0O2+qn8dd9xxz/+DP/iDj65Zs+bI\n6Sz3qle9asLnkuIHb4TijvooRY2DFtsLCzYrj0fKegtbO2owtIs3dq67c3wkgnsjuLvsGWQ6qeVD\nmaypBCnWlndcV5V3rudbZu6yGRnqTo0iZW9bU4YF/J/tBhzUak1aGn4iDcogB8V3tDNjZXOMX2C1\nOj5esdyhMsD4pGaToxsNjmo22WcG71ETeLCddVIeP+6qFJe7Z5L6N1siuH5ggOun2MZQbu+qeFvv\nPJUmNO2eedrjK8veh3rmLEd9zyCEZsMAcEyzyQ/KIMEVAwOsHhtb2J2qqAFv2rqVV61YMe7zjQg+\nvWQJ73vkkW3TDj744GMuvfTSbT8eb3vb25513XXXfW/Od1bqAd1kYrwNOBw4GXg38Brgwsw8f4rl\n6sD1FP0mbwB+ALyiWhQ0Il4MvCEzXxwRJwIfzsxnTrZsRLwPuDcz3xcR7wD2ysyzK+v8PMW53eXj\nBTF29ahcv1u7du0TX/va157/lKc85YVzsf4xipP+dmGlB9rjlWn3lxcgM73DOVdqZbDikLJOxSFl\noc39yhT8+TCdAIIZGVrs2r2xtAMbS2FeL/LHKCrn31n2QNTuiei2en2HbpNn20AZ0Gh3l7xneZxa\n02pxQKvFmmaT3eds6+o1BiHUKy4ZHOTjS4u+0k4aG+PsRx+d0XqabG8SOltur9W4eGiIbwxOnq/4\nyc2bpwx6b926ddP73//+V15++eX/Nou72Hd29Wu+iMj3vvcx+QFz5h3veEf/ZGJE8ctzMfBEivqB\nTwD+NDMv62LdJwA3Zeb6cl2fAU4Fqj2bvISi21Yy8/sRsTIiVgGHTrLsS4Dnlct/GhgGzi7n+w3g\n58CWLvZP44jh4aD4XCwph6UUiQePUpyPj+S6dV1dQS5btmz5WWed9be//Mu/fPpE8zQp2+VT3Nn8\nk49+lNEIrqPIZmhnNoyW7fM7B9jeTWWL7anhm8u08Hb6+JaYeS8TC2HvSlHNg5tNDikzLBaySN90\nAwdmZGixC9hWGHT/Bfi8DgIHljUwnlGZnhQV89uFgDeU9VfamSWbymPvVph2T0ZQ3DG8L4L7Jpln\nj3ZAo+Pvqjnq8Ug7x0CEdgXHVppiXD9OHaGpjAB/u3Qp3xwYoAbsk8nTGg2e1WhwTLM5o+zXW2s1\nPjs0xLcGBro63l5dr/O8RmPSeZYtW7biT//0Ty+99NJLAfjwhz985vDw8D82GlMsKPWRbr5vX8zM\nY4CvTnPda4DbKo9vB07sYp41wOpJlt0/MzeW4xuB/QEiYnfg7RTZG2+b5r7usmJ4eC9gFbBfOTxu\nkvE9KAIXkx5Fh77+dYaAZWWa8TKKk/TqhymAJ116KV8DvthZXJDtxQN7LduhaqAsXjhU/h0oCw3W\n2F5csFZmQSyhKFjYTrteWilSWBUTjA8C+7RaPC6TVa0W4ycT9h8DGVLvCYqT732aTZ4ySaHapGxS\nVwaD2wGOdm887WFTFAVoHyqDIY928Z1/uFbj4Vpt3OYsK8uARnvYv+zSef9Wi5UWJJ02AxBS4aBW\ni6VlnaH7yrpC02nK96XBQYbLTIkWsDGCLw0N8aWhIVZk8oxGg2MbDZ7QarFXWSy98zywCWyo1fhh\nvc73Bge5fpx6ZyeMjXFfrcbN4wRafjowMGUQo9Mb3/jGT77xjW/85KWXXsodd9xx4+te97onTGsF\n6lmL+fg+aRAjMzMifhQRJ2Tm5dNcd7dHhW7e/RhvfeX+taefC3wwMx+JKf6jZY8rbcOZOdzdrvam\nGB7eg6L3mCPGGfaa7e2NRTAGfZPVUFUrU5z3ymSvVou9Mtm7Y3zv8kR5vgsQ9vKBaGeCBXMZyJA0\ndwK2BWVXTuMYMELRbO+h2N7Lzp1ls5Y7yr+TFU19sOx16dpxnhvK5HFlYONxZc8rB7ZaHNhssl8Z\naN7VeTyVZqYOHN5scnVZF+P6ep2TphEQuGWS7I1NEfzn4CD/2dEcZLByU6t9M2+8WkkAT280eNnI\nCE9stdgM/K/ly9nY0QPdj+t1WjDjY93q1auPqNbROOecc9ZdddVV35jh6npKRKwD1i3wbmiedJOJ\n8UzgVRFxC9ubaWQXvZNsANZWHq+lyKiYbJ4Dy3kGx5m+oRzfGBGrMvOuiDgAuLucfgLwW2XNjJVA\nKyK2ZuYFnTuWmedOse89qWzqsRZ4SjkcR6NxPAMDs9p/dK28gB+iOPgmO3a7OJuisq2hzGKbbM9q\nWFaOD7H9gN3uljLKO3LVYbdK9427UVTgX16Z1o6K99Id/MVwQmogQ1o8lkCROZEJrdZjnm8B90Vw\nRxnUuKMS3NhYq014gg/Fb9Ht9Tq3j3MxsaSsu9EOahxYNsM7YCd7ZFoIHi+lufGEShDjhlqNk6ax\n7MrK8ew3Rkc5odHguwMDfHdggHsn6O5+LGLSc+daJsc3m7xsZIQjKuvfHXj71q28Y7fddjgmPlBm\ncZzQZXfvUznvvPOG281Obrzxxh+8+c1vPmFWVrwAypvSw+3HEfGuBdsZzblughgvmuG6fwgcERGH\nAHcALwNe0THPJcAbgM9ExDOBBzNzY0TcN8mylwCnA+8t/34BIDOf215p+aHdNF4Ao9e94x3vuPjZ\nz372SyOilsBttRpX1+v8rF5n73qd+zsPkhN0xdQ2lMk+7SJrrda2SvJ7jjO0u/SbKM4cZW8b7RoW\nWyvtph+JoMmOdSrazSwGyyBEO0ixJGJbzwUDdF8YKTMnDD70UlBiOvrhRHW23tu5CGTYpETqPzVg\nv0z2azY5ruNEvAncW8ncuKsMbLSHyTIARyL4eb3Oz+t1qNwNrZXN9Nr1QdZWghy9Wn9jZ45r/fC7\nIi2UJ1QCBTdMsy7GQZVlr6vXec3ICMc0m/zeyAg31Wr8aGCAG+p11tdqPFLWdGuN833cq9XiyGaT\nExsNntFssscE3/cjWi1+d2SEj5XFSNs+s2QJT3vkkVnvge6II454RjtLY3R0dOtb3/rWE3/xi19c\nNcub0SyqTRA8Wwy6+fw/PM60TVMtlJmNiHgD8BWK69lPlL2LvK58/mOZ+cWIeHFE3ESR5XHmZMuW\nq34P8NmI+F3KLla7eA1dKTMd9gD2BTbnunUbp1hkUqeddto5p5122h8vWbKkq/OkJnDAc5/LpfU6\nV9frXFOv83AXH856per76nJotyXee4L02m5Pcjrnq7G94ueKzB26ToyIyde7kydW1fW3xrm7p9ln\ngEDSfKpTZnFMUK9jM3B3JahxR9nryoZajYcm+L1sRXBHvc4d9Tqd7WL3LgMb7d/ONWXvKY+bx16f\nZtt8HbcNlqgfHVk5rtxUr9Nk4pt3nZ7ebFLPpBnBdfU6d0WwqswKPqLV4ojR0R3mT4qbfiMRNChu\n5g1SnEN368VjY1xTr/OtSmD2pnqd/75iBX+3eTOr5+j7PjQ0tOz888+/8tJLL2VkZOSRz33uc+dd\nfPHFfzUnG5NmoJsuVtcDBwEPlJP2Au4qh9/PzB/N5Q7Otmp3OzE8vIQio+OXgQMoXlv1WPa3uW7d\nGwCe/vSnv2jlypWr3vSmN31qsvVP5+RhDPh5mWXxs3qdawcGpqwzsVsmhzWbHNpscmirxaHNJmt3\nIlV2spOQqU5QphP925mTnYne02ogYyY9Zyy0fjgBnIv3aS5edy/8PyUtrIcjuL3SneyGsveVu2dw\np2qgLCS6phrcKHuLsnvY6emH3zotLmcuX8595XHh/C1bOGQaN8b+fNkyflhmQb9qZITTOgIXc2Er\n8PvLl497Y/NfNm2a94Dru971rlN+/OMff2WeNztti6GL1Q984APztr23vOUtPdXFajdBjP8DfD4z\nv1I+Phl4KfBJ4MOZ2VdtpyIi+frX9wLeSdGbyaQ+vmkT+03jAmmi97MB3FmrcUu9zg31OtfX69xc\nr0/a9hdgRavF0c0mRzcaHN1ocHCrNaOD1XSDFRPNP1XgYi5PVqrvbWcmRr8FMfrlpG6u3qfZfv0L\n/f+U1LtGKHoDuK1WK+pplIGOO6aovzGRfVstDmk2Objyd3Uf1t3oFf3ye6hdx18sW8YPykDE27du\n5TnTKO759YEBPrhsGQCrWy0u2LJlzgoK3xfBDwcG+GG9zvcHxz/CvHxkhFfOQyBlIuvXr7/yD//w\nD49bsB2YhEGM2dVrQYxumpM8KzN/v/0gM78aER/IzNdGxNAc7ttcemDqWQq/t+KxnV0+ZWyME8qg\nwn7NJkMU9SE2RbClVuOBCO6u1binVuPuWo076nU2dHmytFertS1gcVSjwYGt1mMOjhNdrk12IpAd\nzT46n5vqJGKypiI7k80xkelclM7kAnahain008naXL4/s10fw9oYkiayBDis1eKwVgsqFytN4K5K\nYdENlb+PqT9VcW+txr21Gj+sTGs361xbZmsc1GpxUKvFqhneeFhMujl299Nvp3rfAZUbYXdNM1Pr\nWY0GH8vkkbI48VX1+mNq++yMByL49sAA3xoc5LouanZ8ZskSrqjXeeOjj85Z05LJHHLIIU+u9nby\n1re+9ZnXX3/99+d9RxapxXxs7CaIcWdEvAP4DEUNxtMoegipUxQZX3R+OjjITyeIiE7XqmaTIxsN\njhob46hGgwNarccUuqy+yZMFE8Y7ERhv3uleQE60zelkcExXez3jvaadaUaykBbqQDOTgMF8vK/2\nWCJpIdVhWzORTlspsifbGRsb6nVuK7M5xrsh0Yzg1nqdW+t1/qtyfjCUyYGtFoc1mzy+2eTxZfbG\ndNrEa+rfJH9LNB2rKt/5u6f52VkKPH9sjH8fKu7jfnlwcKeDGJuA7wwO8q2BAa6u18ctBgrw+GaT\nm8cJbFw7MMBZu+/Onz3yCE+dxYDKTLz//e//Xru3k5/97GffPPvss5+3oDukXVY3QYxXAu+i7AUE\n+C+KnkLqzGJRzcXk2NFRnj06yjNGR9lznB/m8X6qqxf1U/2YV5t8TJSB0XkBOZ0Lys75ZjMTY6rX\n1k9Bi6qFOMGqvlft8W72Yz7f4+ns11TMxpA0W5ZRyd6oaAB31GqsL5uH3lKrsb5e554J7uaOVnpM\n+Vo5rZbJ2laLxzebHF4GNg41sLFTZpqRqcVpeeXzsnUGn4NTKkGM7w0M8EAEe03zM/gIcPnAAN8c\nHOQn9TrNcfajnskxzSbHNxoc32iwJpMG8N/HyRIHeNduu/HPmzbNeq8lM3X00Uc/t52l8alPfert\n//Zv//aR0a98ZT/gJOAE4J9y3bq+qq3YaxbzcWzKz3lm3gO8ISKWZ+aWjqdvmpvdmnPvBPak6K71\ne7luXQLE8HCdomvmFcA+wDEU71ETWAO8EPiVnd34VUNDXDU0xCfKLuYe12rxuGaT/cu/a5pNVjeb\nXaegdn6Aq5kK4wU0qgGRbj781Xm6CWDszBeqeiE63kV4lResE+umK9qJsnQkSeMbgG1NRapNUx4B\nbqvXubVWK4ZyfLxmKa2IIgBSr/Of5bRambHRztY4vNnkkGaTZfPyqhan6f7eLeaLhV1NNWA4MoPl\nD261OKrR4JqBAZoRXDY42FWBz3siuHxggMsHBrhqgrp4UQYufqnR4KSxMfboeH4AeP2jj3JBR7er\nbefsthtv2rp1QZqWTOaMM8543xlnnPG+j4yO8tWhbdUINgEGMTQjUwYxIuIk4OMUF/ZrI+I44HWZ\n+fq53rm5kuvWvWeC6U3goXK4HbiiY5b3di4Tw8O7A08EnsTIyJP33rTp1N323POIe+p1RqZoZzcW\nwR0DA9wxznNDmRzSaHBoo8FhY2Mc1mxyYLP5mH/YRHefO7sinaggZzWQMVVQY6oAxkQBjcnW2U1X\nqdWgxngBjl7Xqyc+vfQe2rREUj/bjaLrxiM7Urk3A78oMzFuqte5uWyekp03HyrNUb5eTouyzkY7\nW+PxZc9kXfXZrlk32W+mv1/9ZaDyv5xJcV+AF42NcU1ZHPSrg4P81ujoY24+NoEbazV+XAYufj5J\njYsjm01+aWyMZzca7DPF+dmLxsYmDGJcV6/zxuXLOWNkhF8dG5uzoqMztWLH12ZTk520mI893WQc\nfQg4BfhXgMy8IiK6+tBFxCnl8nXg45n52CBAxPnAr1LcyDgjM38y2bIRsTdwMXAwsB44LTMfjIgT\ngI+Vq60Df5WZF3eznzsj163bDPywHADeBhDDwwHsXe7nIcDBg3fffeKhS5f+aqxcuceGZpPNk3zw\nRiO4YXCQGwYHoayCPJjJQY0GBzcaHNhssrbRYG2rxd6VOhqdtSTGC2ZM54JxouDHZNkZky033nzd\nBjIMYHSnn96fqtkIZNikRFIv2R04ttnk2Epw4xGKwEY7qHFzWfy7M7CREUVvKvU6w+W0yGR1q7Ut\nW+PIstaGPaMsrM7fncV8YdEPZuO/8+xGg49nsqks5v/Tep1jmk1uLYt9XjUwwDX1+qTNVQ5rNnl2\no8EvjY2xahrnLgH870ce4c93Gz+kORLBx5Yu5YZ6nTc9+uisvN7ZctuO1wedGf5S17pqNpWZt3Yc\nkKfsi6gs/PkR4AXABuAHEXFJZl5bmefFwOGZeUREnAh8FHjmFMueDVyWme8rC46eXQ5XAU/PzFZE\nrAKujojPZ+aCVLgpm6jcVw4/Hm+eGB7ekyLAcShw2P633fbyA/bf//gbm83YsuyxSaRjEdw8OMjN\nHUVFl2SyqtlkVbPJAdWh1WJlJsFjm5BMJwPjMfs9SQCj2+BF5zITdZnaeWLQTxeonsRMnxkZknZ1\nu0HRdXolsLGVIrBxc73OTZXARmeBv4xgQ73Ohnqdb5bnAkOZHNlsclSzyVGNBkfaDGXBGdToHzM5\nq9wE3FqvU73A+LPddiMyHxOMrBrI5NhmkxMbDZ7RaLDfTpzTHt9sctLYGN+ZpKOBrw8O8rRGg+dN\nowvZudYRdvnpwuzFrmMxH1u6CWLcGhHPBii7VP0j4NrJFwGKgi03Zeb6ctnPAKd2LPsS4NMAmfn9\niFhZBiAOnWTZl7A9/ejTwDBwdmZurax3GfDQQgUwupXr1j1E0WSl3Wzlr9vPxfDwKuCpwNOAp9No\nnMDAwJrx1jMSwS0DA9wy8Nh/57Kyi7eDGg2OaDY5otHgkFaLoY5ARrcmCmCM12xkqnVPp87FbAcv\n5joYspgPKjvLQIakxWYZFEGISmDjUcrARhnUuLnsIaUzsDEawVUDA1w1MABLlmyrr3FQ2d3rwa0W\nBzWb7J/Zc6nli8VsFrHWzqt+Dya7UEjgjghuKL9/t5Q9FE3UBfN4AYx9Wy2eXBbnfFqjMavNwV47\nMsJPBgYmzfb40uBgTwUxHt9sMrw98HLgQu6L+ls3QYw/AD5MUdhyA/BV4H92sdwa4LbK49uBE7uY\nZw2wepJl98/MjeX4RmD/9kxlk5JPUgRBXtHFPvasXLfuLuBL5QBADA/vQxHYOLocjiqHvSZaz9Za\njV/UavxiYIBvlNMGMjm02eSIsn3tQa0Wa5tNls+gHkZn8GImP9BTZWF009xEhX7KVJnMzgQybFIi\naVewFHhSs8mTmk0YGwOKIoTtwMYN9TrXDgywseOCqlpf49uVu7RLyuDGmvI3f02rxdpWiwNaLZuj\nzBOD9L2h2jPgLbUajwJDwF1lb0K/qNW4qV7nxnp90qbf41nVanFk2YTs2EaDVWVG9FzYO5OzHn2U\nD46Tvd12zcAA71u6lFePjLBXJjfX61w0NMQV5Y3PXx8d5dUjI/PWO9JNO9YFuXOeNqtdULe9k7xy\nBuvu9iqim+92jLe+zMyIyMrjy4GjI+KJwJcjYjgzH3rMyiLOrTwczszhLvd1QeW6dfcBXyuHbcrg\nxhETDJ2FjWlEcOPAADd2ZG7sV2Zt7J7J7sDumewN7JPJfsC+FAfMoc4VzuS1TJGFMV4AY7IL08mK\nm843T1Bmhyd7krSjJcATm02e2Gzya2Vg494IrqnXuaZsg3/LBMUDRyK2ZXRQCW7UMlnVanFgx7DW\nIqKzzt+03nBQea67OYIHajVOW7GCWuZjspwmMljJdmp3twywR6vFR7ds6bp3wZ2VFN/rqXx7cHCH\ngGbVpUNDbKzV+JOtW8d9fjbdXKvxzR2vPT4/m+uPiHXAutlcZ69bzMeUCYMYEfE3lYdJRyAhM/9o\ninVvANZWHq+lyKiYbJ4Dy3kGx5m+oRzfGBGrMvOuiDgAuLtzw5l5XUTcDBzOOF33ZOa5U+x7XymD\nG/cB36tOL4uLPg44Eng68AyKZj6PH28999RqE/Z1X7VnJvuWwz7AvmWQYx9gv0z2pWjzNtXXajrF\nOmfSxepCpG8u5oPJXDCQIUmT2zeT5zYaPLdMGW9393pL2d3rLWV3rw9O8PveiuCOep076nUur0yP\nTA5otTis1eKwZrMYWq0d7mKre/6W9Y4h4KUjI3yq0sPHRAGMFZk8oZ29XDbNWpW5LVAxBrxm+XIe\nqtV4uFbjBwMDPHMemm9sAc5fupTvTlITo1uXDwzQhFkJvkx03ra+VuNDS5dWm9x8mVnuXrW8KT3c\nfhwR75rN9au3TJaJ0f5gnUTRZOFiiuvS3wZ+1sW6fwgcERGHAHcAL+OxTTwuAd4AfCYingk8mJkb\nI+K+SZa9BDidorvT04EvAJTz3p6ZjYg4mCIL4cYu9nOXVRYX3VgO32xPLzM3ngEcT9Es5RiKbmK7\nKvT6UAQPRXDzJPMMVbI59gD2qPzdtwx07AscmLmtANlEzUhmEsDonLd6QLWpgSRpVzVRd68PR3B7\nrcbtZbv+DbUat9Xr3D1J+/52cKN6F3efdlCj7Pb1sGaTfecwZV6aC785NsYg8C9DQ9xbfgf2bLU4\ntPx8H9pqcUSzyQFTfLYHgReOjfH5JUWDjK8NDs55EOPWWo2/XLaMu2ZQTL9TLZPnNRqzVi+nM4Cx\nMYLPDQ3xtcHBaqBoDHhjeZ2inbCYg6MTXrRm5qcAIuIPgOdk5lj5+KPAt6dacRlMeAPwFYrg3icy\n89qIeF35/Mcy84sR8eKIuIkiqHjmZMuWq34P8NmI+F3KLlbL6c8Bzo6IMYovx2sz8+Hu34rFo8zc\n+HI5ABDDw0MUgZ8DKGps7EWRXHEARZ2SA8u/B8DUx7rRCO7//9m78zi5yjLv/5/rVFV3OjtZOxtJ\nCGGHsIMImpFlEBBUGBlUBsEVBJ9ndFRmnJ/bOCMuo7Ko44AL484oo/DgKCBElE32fYdgCEkISyBb\nd1fVuX5/nLs6pytV3dVLdVd1f9+v13lV9amz3NV9uuqc61z3dQMv9/HPFbmzszt7FYvsFcfsUSwy\niaEv8DkcgYux/EFST42SjaFaGyLSzCa7b1c8FJI6G6tLwY0wKsqq8HOlu9MvRREvRRF3pOZNChd/\ni4vFpCtKqL0xWZ+Z0qAMeEs+zwn5PJ0kwYiBZiIclQpi3JvJ0Al1qzHxcCbDv7S1sTn1v3lcVxcH\nFAr8S5UhVyu5YMsW9igWKVDjHcx+WG/GLdkst2azPJrJ9PgcybhTNDvXly9/fIh3K2OM9XVSbmaP\nAYe5+0vh52nAre6+6zC0b8iZmbv7yF8RNSlbsSJLUkx1Hj2DG/PL5g14hLeFYei5veKY3QsFpleo\nldGIGuFCu5F/P4PVn9/vUP8eFMAQkbGmk+SO71OZDE+H6dkooqsfn8VTSoVEywqKzhyDI6U0wjmC\n1M+Hx49nVaiN8f9t2cJBxaEfIPGuTIYvtrV1/w+2uXNeRweHFwrEwDkTJvB8P7IzTu7s5IyuriFp\n2wYzbsxmuTmX4/EqdXn2KRS4H/b0o456eEh22ofRfs1nZn7JJZcM2/7OPfdcGun3WUvw7QLgbjO7\nkSRw+Ubgs/VslDQuX768QFKfZHW1ZUItjglsy+iYBt09SGaxLdCxGNi9fP1nQ2Gy34SfZ4YToInu\nZNzJkhy4pec5kqrrLSQf6NPcmRHHzIpjFSUbRRolI0NEZCxoBZbGMUvjuHt0lCLwXBQlQY3w+Ewm\n0+OucNqrUcSrUUT5FUuL+7bgRsjemB9GShmuURJEhtKBxWJ3EOPebHbIgxi3ZLN8ddw4CuF/bUoc\n8wbW7jIAACAASURBVNmtW1kSul5HwPs6Ovh8P7Ixftnayn3ZLF/bsqXPZdeacW82y5r77+eZe+4h\nn8nQUiyy53778aODD666nrmzT7HIO7q62LtY5KSTTnoEjTo4apjZVOAykvIETtKr4gmSMhQLCb0m\n3H3DUO+7ltFJvm9mvyUZ4tSB891dQ+JIVaGP26YwreptWVuxYhrweuANwBEkdTp6hHBrLThayfxQ\njGmXUM19YRwPW9VoGXojEchQFoaISCIDLIxjFsYxfxXmOUm/96czmaQbSibTXXujWtZGlxnPhABI\neqQUc2emO3PjmPkha2NeCHBMa/K6GyNRbFyGzz6FAv/Tkozf91CVTISBujGb5cJx47q7ZcyKYz6/\nZQtzy85NDiwWOSKf54/9KPb5ZCbDn7JZDisU+ElLC7dks0xz58h8ntcXCtyQy/G/uVzyv3rffXD/\n/fDBD3av/9Cll0JrKyxbtt22P9zRwcGFAjuEdp599tm7xXGsE6oh1ACfJxcCv3H3U8wsS3IT+1PA\nde7+ZTP7JHB+mIZU1e4kZrbE3Xur3VjTMo1mtKcWNTtbsWIicCjbghqHAuN6XakfWsOwWPPC3Z+5\n4fnsOGbiILfdAB8kwOjuUlLS1+96KH8HCmKIiPRfTDL863NlgY3nQnZGf00KI6UsKRa7p/YmC2w0\nynmCDL0twDsnTiQ2w9z5yaZNTBjkNovAz1ta+HlLS/eoHvNCAGNmlfOSV8z48IQJbOrnsbZTscjT\nfQVfvvtdeO97t5//ve/BWWf1mHVkPs//6ejo/vkLX/jCSbfffvtV/WrUII32az4z829+85vDtr8P\nf/jDPbqTmNkU4B5336msXY8CbwyDdbQDK9x9t6FuT2+ZGP9mZhNIRgO5E1hD0p1kDsnd8hOBjcDf\nDnWjZOzy5cs3AdeHqVRwdGeSYWInkPQeSU8tJEGONpKi7FOAebgvAJZi1uNMqdOMpzIZniq7+wMw\nwZ1ZIaAxO9xpWlIsskDZGw2nr4yMoQo86IRTRGRgImCWO7OKRfavMFLK6lRQo/R8XZViogAbo4j7\nooj7sttOXSe4s1MqqLFzHNMex2Ou3oaMvPHATnHMk5kMbsajmQwHDKJLyVozvjFuHA+njvdFxSKf\n27q1O7Ohkh3cOaujg4vaqpemy7pzSlcXP2vd1nmrzwAGQLVlKgQlTxyiWhvSu2gIRqgZhMXAejP7\nPrCMZGTT/wvMdvd1YZl1JLUUh1xvo5OcamY7kwQp/pWkXwvAsySjk5zn7k/Xo1EiJb58eRfwcJj6\nxVasGA/sS9IV6hDgdcCO1ZbfnE5vTcm5s2Oouj4vjml1p5WkT29LeHQzYpJxxmNgvDuT3JOhZcPy\nMrTq3bWktG1lYYiIDK3J7kwuFtm97CIvD6wJQY309FyVuhubzXggm+WB1IXe+BDY2DkM/7o4jpkb\nx0M+AkN/KSg++u1SLPJkOIdcOcAgxibgv1tbuTqX665/AbB3ocD5W7cyqYZtHFko8KdCgbuzlY/6\nghnL83kez2SqLpP2xnyercCfq72fUOPibzs7ObWra7sbf9///vc/MdxZGDIsssD+wLnufoeZfYOy\nbiPu7mZWlxPpXo9cd38S+EI9dixSb758+RbgljABYCtWzCTJ6khPO5NEEyuGrfPp7I0Bagn9fOeE\nvr27hRFYpgzxBfJYu+CuVyBDAQwRaXR5YIsZ6ZJ8rcAU96bNHswBO8YxO5YV/ivV3Sh9Fz+VyfBU\nFLGxwl3ILWY8mM3yYOriLBtuRiwqFlkUHpcUi4PuRlorBTDGhvRxu6qfd8i3AFe1tPDrlpYeAbvI\nndO6ujilQnCgGgPO6+jgIxMmsLHKsfehiX0f/R/q6GBGHLN7scgZEyfCgQfCZZfB+963baFLL6X1\noIM4s6ODN+fzTdW9azSo52fLY489xuOP9zoS7nPAc+5eGnH7F8A/AmvNrN3d15rZHOCFerSv7oFp\nMzsW+AZJPajL3P1LFZa5CHgzyf/we9z9nt7WDcO8blf11MyOBr5I0sWgC/i4u99Y33cozcSXL18P\nrCfJJuoWRlSZRRLMWAwsBfYDDgAWDHa/XWasDumzd6bmzwvBjN1D4dH2Jj75HClDHcjQyaYMhQJJ\n3+SXo4iXzXgpPC89bjUjT/JFFZsxJY6Z6s5Ud6aHLm1z3Jkdx0xpsn7/MnBbgBejiBfNeDEUtX4x\nHDObzdhKkn2w2axq0cwoHEel0T8WhG6RC5r4WDKg3Z32QoHXFwpAEthYHwIbT6YCG69VuHgshMKj\n6ZR5c2dRHLNXocDexSJ7Fgp1CWroO2XsWJAKYvylxiBGB/D/Wlr4n5aW7QIOuxaLvK+jg10HMJrH\ndHfO6ejgS710K+nLcWFUojVmSVZIKN658NJL6chkyBSLvH7ZMt62ZAkTw7Llrr322kuvvPLKrwy4\nETJidt11V3bdddfun6+55poer4cgxSoz28XdHweOAh4K0xnAl8Ljr+rRvroGMcwsA1xC8qZWA3eY\n2VXu/khqmeOAnd19qZkdAnwbOLSPdc+nctXT9cAJ4Ze6J/A7kqE8RXoVRlRZF6bb0q/ZihXTSfp6\nLSM5ntoqTDHJdUsh89prU3Zva1u+MYrYaMZrpQ//ClZnMqzOZLg2/JwJGRuzwhCxs0t1OsLFzA7u\nFfv6jvWMgaEKZKS3MdZ/p1LZFuCFKOI1MzaGi8mNIUjxYhTxYghUvGrWXYitFmt7OeEdF7K42kNw\noz0MRTk7jpkZhpqW5rCVpLtEaXohFbB4MQQqBis24+UQ+Hig7LVJoZj1rPDdMtWdNnfGuSfFpcLz\nNpLjrjS/EWtMGKHmRqHA61KBjRfNuoMaK6OIlZlMxRHOPNWF9GqS4M8+YXSHQ/P5IQloKIAxtqSD\nGM9FETHV/3c6gd/mcvyipWW7Qrfz4ph3dnZyeKEwqKDjsvB/MRBnpYpyznZnahyzIYpg2TLOXrqU\nPYZ4CFlpWucBPzazFuApkiFWM8AVZvZeQrJBPXZc73Ofg4En3X0lgJn9DDgJeCS1zInA5QDufruZ\nTQ2VTBf3su6JwBvD+pcDK0iGfr03td2HgTYzy7l75fCgSA18+fKXgBvC1G8hy2MSIcNj0bPPfnLc\nvHkHPpnJbBfcKJqx1qzqBU3WnZmpoEZ76oKmPY4r94cZI4YyI0MBjLGrSHIRtCaKWBsKDXZPZhXT\n1+uto0q9HkguvGa5VwxwtMfxgC9AnaS7QocZneGxi+TMJEfSRS4HTFQQZTudJIGK50OgIv34yhAe\nP5E7E0LAobTVLVAxE6FkYxTxaBTxaD/3VQpotIV6T1NT06zSiF9xzOQRzvQwYKY7M1OBDUjqDKwM\nQY3S/9IzZUVEYzPuzWa5N5vl2+PGsW+hwOGFAofk84wfYHtGYlhwGTlT3JkUx2yMIjpCcLt8FJE8\ncG0ux3+3tPBy2f9qexxzamcnywuFQWflrjfjH8cP7Mh9f0cHb0llVkTAwYUC14YhZO/IZGoKYjz0\n0EM3XXzxxR8YUCOkJiP9+eLu9wEHVXjpqHrvu89zD0tGd3gXsNjdP29mOwLt7v7nGrY/D1iV+vk5\nkgKLfS0zD5jby7q1VD09GbhLAQwZaSHL4zXgvjD9AsBWrGgj+cd/PXA4yag/s3rbVsGMNZkMa6q8\nPjUUMJsfUod3LxTYaQyNrjKYE8aR/iKQ+iiSXNhtNmNTyJwoTa+ascGMV6KIDeHu9QtRVDVzqj/M\nnSnuTAvdQ6a5My2OmR7mTXDvLg4MbGuLGetD0GRtuPjt6KU9cR+Bz1JbMiQnopnSFLK60j/HqYBF\nZ9h2LSa4Mzl0VSgFVNpDmv7sURoQ7IDuv08pSFF6Xn5h0h85d2aEDJsZccyMELieHo6Z8anHcVAx\nYJAnDG2aybAqivhLFLEqDHHa27HU6/s1o8OMDVD1+weSY2FescjscNxPTx3/k9yZGKbhDnxNBPYq\nFtmrWIRwcbYFeDST4YFslvuz2e6CjJB8196Zy3FnLkdu3Dj2LxR4fT7PQYVCvwMaCmSMHUaStbAx\n/Lw+ipgZLvaLwB+yWX7S2soLZZ8RM+KYU7u6ODKfH7L/jUsr7CctcueIQoE/lI3U9++bN7O0QveV\nA4vF7qzhu7NZztDoIzLCavlf+RZJqvybgM+TBLS/RXLB1Zdaz15q+XS3SturVPU0dCW5ADi64obM\nPpv6cYW7r6ixnSJDxpcv3wrcFCage0SVhSRZG4t6TO6LMZvR2zY3RBEboqjHUC4T3DmgUODQfJ79\nC4VRn60x2BNGZWGMvCLwghmrMpnuDIhOM4zkQjybuijvIinktzVMW1I/lx6HUjbced4hdUE2MRWs\nmBEuPnfo54VatX6PTjIc5dooYo1Zd3CjNNVywexmbJdUPIS/l81mbA7B1fI7/IuLRQ7N5zm0UGBh\nHDdNPYYC8FL4fa+PItanfvdrBpFRkQlZdHNLGTMhSFEKWAxFzYocMMedOYVCj9tjMcnd2bXhPb0Q\nujx2sC1I0QHd/0ul5539OFY2m/F4NkuvpeBIuq1MdmdKCH5NDu99cgjQtIYgzbjw80SSrJ8JQ1g3\najywf2n4185O1ptxcy7Hn3I5nkgFNPJm3J7LcXsuR0v4Pj0sfJ/W2uWk/HtFQY3Ra2YYZhWSYrR7\nAI9GEd8aN46VZdl00+KYv+nq4ph8nlyFbQ3UejNuSwUnPrZ1K9Pd+adUZsaiOOa5ss+xr2/ezJIq\n9Tf2KRTIuFMMmYGvmvVanH7Dhg3rzj///DdWXaAOzGw5sHw49znSxvJnSS3nWIe4+35mdg+Au79s\nZrX+r62mZ1HEBSQZFb0tMz8sk6swf3V4vq5a1VMzmw9cCZzu7s9UapS7f7bG9osMqzCiyiP07HLV\nzVasmMi2IMeSHpP7TlT439xsxk25HDflckThBHp+qP6+uFhkp3Ay3Yh9noeLRiMZOVuAx0N691+i\niGfD3eNqRQuHw9RUF632VPet3urS1IuRpChPKRbZtcLrnbBdYGNt6kJ1ML/HTOpCsjS0dJHkTn9X\n6F6yqY/aH6XU/Z8CU+KYvYpF9igUmBvqMbQArWWPg/39FthW/HJzaOPmENTKk2SYWAgWlO6OvBwC\nFS+Ei/tXzGrORClXClTMCdPc1PNZQ3gR3l8RyV3i2cUi9KM/e5HkOCsFN14ty2IqDYn6fD8yPUrb\nWjeAYFBbKng4oTxDhSRw3xY+y0uXY56eQhvTr6Uv247I59mzUOC+bHa7LlxdZtyay3FrLkfGnT2L\nRQ4uFDgon6e9H98fvX3XjOWLktFgxzjm1vD8jyEgdk0u1+Nzckocc3JXF2/O52mtQxteKjuGlhWL\nTHXnfR0dXDZuHECPIrcAp3V2Vg1gQBL0W5wK0Pwliti7l8+R008/vX2AzR+wcFN6RelnM/vMcLdB\nhk8tQYyuUGQTADObSc/P+97cCSw1s0XA88CpwGlly1wFnAv8zMwOBTa4+zoze6mXda+iQtVTM5sK\nXAN80t1vRWSU8eXLN7Gt8m8PtmJFBtgR2AXYnWRklSOBOaVl4lJ3lEyGO1LrjnNnUahgPzsUFZ3l\n20ZGaLbuKP0JRuiEcXhtBB7OZnk4k+GhbJanyvqlDzVLXdiUT5Pd2aHUtz8EKGbG8YD7v4+EVioP\nSZkWk1yIlh4LZt3Pu6fwNyjVPmiFmu4MFkkCBq+FrjClgMrTUcR92WyPrjmvRhE3RxE353rf8rjQ\n9WaHkOEyNdT7aQmfRVtTAYpK3YQG2mWiP0oZOXMqBCtGW7HVDMkFzHh3cO81a+hlS0bhejGMylMq\ndPtKFLEZuovhDuZ/vhQAWT/gLQyNohn3h64ol40bx44hoHFwocDSYnHAwbjBfH/pu2/kHVQo8PPW\nJDRxZ7bnJ0GrO2/r6uKtXV11/Z5ZHMdMdGdT+Bt/bPx4zgl1Lp7IZLbrQvKBjg6OrzK6SNqCVBBj\nVR9BDBke0QjU6WoUtXzPXgz8DzDLzP4NOAX451o27u4FMzuXZJSQDPBdd3/EzD4YXv+Ou//GzI4z\nsyeBzSRVTauuGzZ9AZWrnp5Lclf6M6no29Hu/mIt7RVpZr58eRF4Jky/g+6iovsCbwvTnlTovtVh\nxqPZbNVib9lwR7Y1lebblkr/Td8Jm5i6MJwWLj6a4VRJWRhDbwvJ3Z7S0IdPZTI8V6E4ZSVTw4V5\nqUjlRHdiwjBAZt0X3y0kF1fjw93X0sVWW+rnsfsVn4go+x1UOtYHePxngMnhc2A+9Dip3UJyEn9b\nLsf9mUzNRVE7zHg+k+H5AbVo6OyQCujODM9LAYsZTRjcrTcjGdZxeh+ZHjHJsfFqGOXn1RAEK2V4\npLu2dIQMmk1h2gL9GvVnOP0lk+EvmQy/aG1lShxzUKHAQYUC+xYKjKvTPgfzvaUuLvWxSxxzcD7P\nn8sCBfsXCpzT0cGsYTjXaAXO6+jgi2F41fVRxOfGj2fvQoG35PPbBTHeUOMoKPPLRl+p5rTTTtth\n06ZNA2m6SM2slg9AM9ud5I4uwO/TQ6Q2GzNzd9cntYxJoeZGKVNjb2C/MFUqjjskWkNQo3RndYdQ\n5K00r1TwcALDH+zQkKq9y0N3TYkopN+XLogNuutSbAVeDGnlz4YhDZ+JIp6vIWBh7iyMY3YtFllY\nLHZnFfTW11aaTwysjCIezGZ5NhwrW0IB0c7QNaXTbEi6EZVG7ZhAzwyc8aXuKu5g1qOLwWTfNrT1\nrBCkGMo+6jI0SgGQTenARphKXYY2kwQ/St2FYFsQrzSv1J2o9Fr6qCuSdBvZFIIrpZo0/akNUm5W\nHPO2zk4OKRSY3gSfbQpoDM4W4JvjxnFXNkt7HPP2ri6OGORwqQNxWzbLRePGdWdk9OaUzk6OyueZ\n28vxeWM2y9dDYOSN+TwfSw3DmhaCGBsG1uqhM9qv+czMv/vd7w7b/t773vfSSL/PqkEMM5tWPis8\nOiS1MerYrroZ7Qe0yEDYihVzSDI2lrCtmGipuOgODEN8YVzqbmepOGKpr2g2pLeX7q63VXje3wuO\nwaThNhInSdFeE2ohrAsjbWw2Y2M4Ed8aLtjSJ/WVnsfQozjmUIzSUS7jzpJikT3DtHs/iuPJ6Ock\nKZkvh7oUpccukgvLAkmmTXmAIl0fYaDDyopUE5PUGVgVRawOdXuezmR4doB1Z6bHMe/s7OSIOtVE\nGAoKZIwOr5jx85YWfpfLdXcb7M1Ht25leWG7UtAA3JTN8tUQxDgin+fjFYIY//AP/3DoY489dvvg\nWj00Rvs131gPYvTWneRu6D7v3RF4JczfAXiW5AJHREYBX758DVVGzgtdUlohuW4I03hgKjATmBae\nl6bpQDtJLY45UFsmbYclo1GsqrG7QblsuIBJd2WZER4nhyBIa+jv30LSDaGVpGDdhlQ682vhoj8X\n+rW3ApPCNiaFtPnJXn14w3JdJCe/L0YRRtKvvzU8looZtpJ8GMdsqy+wKQQhSn3/y+82biEZjWZt\nFLG5QU82I3d2jGOWFIvJFIrJNupJu4w8IxkOc2IcsyP0qwClSL1EwEx3ZpZGMwmKwOoQ0Hg6dJl7\nOpNhSx+fyS9FERe3tXFxuCA8MJ9nUairMi88TvLqI9WUMlJeC11yNqa+N7pIRlQpkPw/5ULXpwxJ\nnZtM+G4rZdVF3nMI5u5HdzJm3fMqLufevZ3t1i2bX57x0h+l7oMFtnUnzJB87+dIvj8b81tw5O3g\nzoc6Ozmpq4v/19LCDblcr+cMX2trY/7mzexcoc7SxtR66k7XGMZysLFqEMPdFwGY2aXA/7j7b8LP\nbybpWy8iY4AvX+6QdE8GXurPuiEAMoUkmFEKbMxlW4Cj9HwBDG7010KpT/VgNtIP2VRAY1JIVS+N\n3lAKPrxkxoYa6wBYyARppP7emZDpYu7EoRhkqZJ/9xQCPjND0GhBscjiOGZR6BqigIWIjFYZthXW\nXR7mOckQ0U+HgMat2Wyfwfk7cznurHNbG0FUKegR5mXYlhGYZ1uwogB9FoI1T4bhnRbHSV2W8DjD\ntw19PX2Euq02ijnuvL+zk3d3dnJjLsed2SxrzZjuztOZTI8AxUcnTOA/N22i3b1HlupjqfOZJRUC\nzF//+tfPaJQsDBn9ains+Tp3f3/pB3f/XzP7Sh3bJCKjRAiAbAhT1Vo6IdgxjSTrayFJUGM+yY2j\niFC/EZhUYZocHof1xkDBjJfNGKp+dYMJXoxz7x4OtD3UGZnk24YhHB/u6JVORbzCBMnJXbowZgt9\nn/Cl1xURGeuMMJRtocDrCgXe1dkJJBl/D2Qy/LK1dbuhW8eKUjC8hyEI3LsZG4GNmQzP9rJcqzvT\nQ4bmdPfumlylwMfMkM05mv86bcBx+TzHpUYjceCaXI7/HLctcfYDEyfy75s2sXMIYrxsxm2pgqC7\nKktORlgtQYznzeyfgR+RfDa/E1hd11aJyJgSgh0vheme/q4fgiDjSAIhpYyPeWGaT5INUuoGMyEs\n2xamDmB9anoReI3k87GFJEAyPUwzUlOtBedjks/M50huKo1P7bsNGI97G2Y5oDvwUB6EKBUmTNcD\nmBiCF1N7STuuNwUvRET6NtWdIwoFjgj1BjaYcXVLC79obdx8tfY4ZnocUwzBh/RwzTF0zy+WvVY+\nv5S1N1AWusCUpihkB+ZJsjZqqfUASfHg5814vpcMyUwIdMwMQY1Kj4NKG21ABhzf1cVPW1p6jCL1\nyQkTODqf562dnfx3a2v38NULi0V2KetucsUVV3zhhhtu+K/hbLeM7e4kfY5OYmbTgc8AR4RZNwGf\nU2FPERnLwkgvpcDGdJJgx0SSQMXLYXoBWO3Ll1euktVzexnoHiq3rk455ZTzTz/99C8ARFGUadai\npiIizS4GHslkuDeb5YVQhLnUvaLAtvoW+dRFez7UwXi5TgWY0+bFMcsKBfYtFtlrEIWY010RewQ9\nQreR0mvltTuy9J1mWQReDV04XzbjpSjipdTji+F5xxD9riaWghpVAh07hC4yja78u38rSeDi2T4y\nhT63eTP7pYIYN998839fcMEF76hHGwdjtF/zmZl///vfH7b9nXnmmQ1V2LOmIVYHvHGzY4FvkHz+\nXObuX6qwzEXAm0lqFL3H3e/pbd0wasrPSVLOVwLvcPcNYf4vgQOBH7j7eVXaNKoPaBGRRjFr1qwd\n58+fv/vpp5/+bzvvvPP+pfkKmoiIDF5MUoPppTCSz8upi/iXzXgliugk6f5YoGdQpEDtGQwlkTtL\n45i9CgX2KhbZrVhkwpC/q/pwkguNUmAj/Xt6ORTfXm/GqzXWsepN1p1Z7uwaRuHaq1DodejS4dbX\nd/AaM746fjxPVghmHJDP8+mtW3tkACiIMTLMzH/wgx8M2/7e8573NFcQw8xurDDb3f1NfayXAR4D\njiJJpb4DOM3dH0ktcxxwrrsfZ2aHABe6+6G9rWtmXwZedPcvm9kngR3c/XwzGw/sB+wF7KUghojI\n2HLUUUe9Z/HixcuOP/748zKZjDJMRER6USqiuTWMelXK7lgdRTyYzfJQJtPrELLmTnso6LxjHLMg\ndG/s7gZJ0vWxv0Ogj6RO4EUz1oegxvoo6vl8gNkvc+KYAwsFDggBoJahb3qf+vOdmAfOmTiRF8qC\nOmd0dPD2rq7uIMbKlSvvP++885YNZTuHymi/5hvrQYxaamJ8PPV8HHAySQC3LwcDT7r7SgAz+xlw\nEj2L+50IXA7g7reb2VQzaycZvrXauicCbwzrXw6sAM539y3AzWa2tIa2iYjIKHP99df/IDz9++HY\n31e+8pVbp0yZMnPOnDlLQBkmItK7Ruu/Xhp6dRzJUJyUPsOKRf4mn6eL0NUlk+G+bJanoqhHEWo3\nY40Za6KIP/eyn5ZUjaeJIbgx0Z0ZYTjZOe7MjWOmjGB9p5JWYJ4786oUroxJuq6sLw90pB5fq5DN\nsSaKuLqlhatbWhjvziGFAofn8+xbLNY1yDOQ7yUHvt7Wtl0AA+Dnra28Lp9n7hC0TQYvGoLMoWbV\nZxDD3ctHffqTmd1Rw7bnAatSPz8HHFLDMvNIhl2stu5sd18Xnq8DZpc3uYa2iYiIDMrHP/7x1w3H\nfmbPnr0wl8u1futb33qsdBGkgIlI8+nv/+1IBz1agGXFIsuKRejqYiPwQDbLw5kMD2UyPBNFNRXs\n7KpxNK/x7t1FI3ctFtm1WGRGAwQ20iKSgM8O7tsVtyzpAFZGUfJ7ymZ5IJPpUY9jixk35nLcmMsx\nwZ1DQ0BjWbFY093lWgzmO+LXLS3cnBqJZEmxyIuhq02HGZe0tfFvW7eSz+c7GzULQ0a/Pv9XQq2J\nkoik5sTkGrZd639PLZ9N6dEBt+3A3c2s3/+lZvbZ1I8r3H1Ff7chIiIyHNatW1caNXBYzuXf8pa3\nnDd37tylxx9//HkKmoiMnEYLekwCDisUOCyMsNIJrI4i/hJFrIoiVkcRm0LXlE1mbAE2mdU8MskW\nMx7JZnukbE+NY5bEMUuKRXYKj7MaLLBRbhywWxyzWxzz9nyePPBQJsNd2Sy3Z7OsTd0932zG73M5\nfp/LMTFkaBw2iAyNwX5WdwI/So2Y89ddXXyoo4Ono4hPTJhA0YwHs1lWmzFrUHsaema2HFg+ws0Y\nViMd6BxJtQT87mZbAKFAUkzzvTWstxpYkPp5AUlGRW/LzA/L5CrMLw3rus7M2t19rZnNIan+3y/u\n/tn+riMiIjIWXH311ReHpx+p977mzZu39P3vf/+Fc+fOXdre3r5zvfcnMpoN5AJ2MBdBrcBOccxO\nVTISILmA6CAJZmwyY3N43AisiyLWhODHmihia4W2bIgi7ooi7spuu2RpCSOBTEt3U2Hb0OMTwjDl\npWmiO+NhxEYMyQH7FovsWyxyVmcnT0cRf8xm+VMu16PLxqZUQGO8OwcVCrwu1NHoayDeoQo0v2JG\nPvV3eH9HBxGwII6Z7s4L4bUXoogPvf3ttQ41PyzCTekVpZ/N7DMj1hipu1qCGLu5e0d6hpnV5MKm\nUwAAIABJREFUctDeCSw1s0XA88CpwGlly1wFnAv8zMwOBTa4+zoze6mXda8CzgC+FB5/VbbNsRuS\nEhERaSKrV69+AjhuOPYVRZGddNJJHz3rrLO+CsouEYGB/x/UGvwwoA1oc2dmL/ty4GUznowiHstk\neDyT4YlMpmJgo8uM1ZlM993NmtoRAhkTQpBjgjsZtk/zttTU28+EYVTHAVPcmeze/Tg1FDjdwX27\n4INBklnS1cUZXV08EUX8KZfjT9ksL6YCGlvM+EMuxx9yOVrdOaBQ4PUhoDG+9Durw2fYDHda3LsL\nuv4hl2Npsci/l9XImN1L4EqGz1jOxKhldJK73X3/vuZVWffNbBsm9bvu/kUz+yCAu38nLHMJcCyw\nGTjT3e+utm6YPw24AtiR1BCr4bWVJNluLcArwDHu/mhZm0Z1pVoREREZWTNnzlywcOHCvU877bRP\n77LLLocoYCLS/wuumGS4z6cyGZ7KZHg6ing6k2FjE124tYXgRjZ0gYnYFgiJUo8AT0VRzcPeHprP\nc2xXF1PdKZL8ropmyWPq50wIpLSGxx3iuDsIUs33Wlv5dapLibn3KOh6bFcXd59++uJ169atrKmx\nI2S0X/OZmf/oRz8atv29+93vbqjRSaoGMUJXjbnAj4F3sq0uxWTgP9x9t+Fq5FAa7Qe0iIiIjC1H\nHnnkGUuXLj3wmGOO+UA2mx2J0RtFhly1oMdmku4Mr4auKd3dVNjWbaU0bQxTpYyOsWqcO9NCd5wd\nQubIOJKLvM1htJl7s9sn67e4877OTm751KeOveeuu3437A3vp9F+zWdm/uMf/3jY9veud72roYIY\nvXUn+WuS7hrzgH9Pzd8I/FM9GyUiIiIitfn9739/Ocmw8+cNx/6+/OUv3zxt2rS5s2bNWjQc+5Ox\nqdqN1vHAoipDoFYLfBSBLSQX6aW6HKWtl9bwsokqz0tZCTGwlWTI1VdDUOU1MzakpkIDBk86zHg+\nk+H5fqwzr1jk/K1bWejOrcoskwZQNYjh7j8AfmBmJ7v7L4evSSIiIiLSqD7xiU+8fjj2097evjgM\nL/yIuuRILaodJxEwEZhY9no9awo4sAl4LYzQ4iSBj/Rjj3lhmR7z3OkMgZfNZjyeyXBTrvZxS/Yo\nFCiS1BHZEoa6zQ/gPV+wZQuT3fnPSy/9yN13393wWRgy+lUNYpjZ6e7+Q2CRmX00/RLJ6KZfq3vr\nRERERGRMWrt27TPh6bDczj7ppJP+fvbs2YtPOOGEYclokZFXz2FsjaRQ3yR3GGQQLt3Oj23dCsBa\nM27J5bgll+OJTKbiequiiOX5PEfn8yyM4+7AystRxCtmvBIe82wrwjrBnQvb2rq38aGtW5msIGJD\niqKRGnNn5PXWnaRU92US2xfvFREREREZNX79619/PTyt+/DC7e3ti88555zvzJ49e/GcOXM0vHCT\nqGfQoz/bcXfa3Xl7Vxdv7+riBTNuzeW4OZvlsVQ9i41RxNWtrVzd2srSYpFjurp4QwhoLASo0C3n\np6minlPimDfk8wBcc801l6SG4BYZUb11J/lOeHq9u/8p/ZqZHV7XVomIiIiIjFIhy+SYeu/HzCyT\nyWSPO+64c973vvd9o977k576E/ToV5ZH2bKz3Dmpq4uTurpYZ8YNLS38PpdjfepO/ROZDE+0tXH5\nuHH8dVcXx3d1Mb2sfVuAK1u21QZ+d2cnEwbQPhkeY/lv0lsmRsnFwH5l8y4Cahli9Vi2DZN6mbt/\nqcIyFwFvJvm/eY+739PbumGI1Z8DC9l+iNV/BM4iqd/zEXe/tob3JyIiIiIy6nhyFZ0HLgxTXc2c\nOXPBTjvttN/JJ5/8yd122+2weu9vNBlI3ZfSRWz6YrYdeGdXF+/o7OT+TIbrWlq4PZvtLjK6yYxf\ntrbyq5YWjsjnOaWriwVxDMDD2SxdYbn5xSJHhiyMe++997r/+I//UDcraRi91cR4HXAYMDPUxCj9\nd0wiCSz0yswywCXAUcBq4A4zu8rdH0ktcxyws7svNbNDgG8Dh/ax7vnAde7+ZTP7ZPj5fDPbAzgV\n2INkRJXrzWwXd4/78wsR6S8zW+7uK0a6HTJ66JiSoaTjSYaajimpZv369auAVcBV/VlvoMfUm970\nptN32223w970pje9p6WlZVx/1292vQU+MsB+xSL7bd3KxijihmyW37S0sDZkZxTNWNHSwh9yOQ4r\nFHhHZyebUsGQ5zIZCtRw0ScjRpkYlbWwLWAxKTX/NeCUGrZ9MPCku68EMLOfAScBj6SWOZFkSDDc\n/XYzm2pm7cDiXtY9EXhjWP9yYAVJIOMk4KfungdWmtmToQ231dBWkcFYTnIcigyV5eiYkqGzHB1P\nMrSWo2NKhtZyBnBM3XDDDT8EfgicPcTtqeiCCy64adasWYtmzJixYDj2N1QmxTEndXVxQlcXd2Sz\n/LqlhYdD7Qw34+ZcjpsrjHryjsmTueypp/7y6U9/uu5dn0T6o7eaGH8A/mBmPygFE/ppHkkktuQ5\n4JAalpkHzO1l3dnuvi48XwfMDs/n0jNgUdqWiIiIiIjIoJx//vlvGI79zJkzZ0lLS8u4iy+++MGh\n3G4GOLRQ4NBCgccyGa5oaeHOPoZsPbtY/Pf3DmUjZMgoE6N3W8zsqyTdNErj7bi7v6mP9Wrt2FXL\nb98qbc/d3cx6249GVRERERERkaaxZs2ap8LTul6lfhywFSv2m/Dqq9/ePGVK+c1mAPK77LJvPdsg\nMhC1BDF+TFJI8wTgg8B7gPU1rLcaSKdaLSDJjuhtmflhmVyF+avD83Vm1u7ua81sDvBCL9taTQV9\nBD5E+s3MPjPSbZDRRceUDCUdTzLUdEzJUNMxNXI29/7ymQZnDk9LpD+UidG76e5+mZl9JNXF5M4a\n1rsTWGpmi4DnSYpunla2zFXAucDPzOxQYIO7rzOzl3pZ9yrgDOBL4fFXqfk/MbOvkXQjWQr8ubxR\n7j52/9oiIiIiIiIiTayWIEZXeFxrZieQBBV26Gsldy+Y2bnA70i6YH3X3R8xsw+G17/j7r8xs+NC\nEc7NhChftXXDpi8ArjCz9xKGWA3rPGxmVwAPAwXgHB/IWEUiIiIiIiIi0pCsr+t8M3sL8EeSrhoX\nA5OBz7p7v4ZOEhEREREREZHBMTP/5S9/OWz7O/nkkxuqR0PU1wLufrW7b3D3B9x9ubvvDywZhrbV\nzMymmdl1Zva4mV1rZlOrLHesmT1qZk+Y2SdrWd/M9jGzW83sQTO738xah+M9yciq1zFlZovMbKuZ\n3ROmbw3Xe5KRVc/PqfD6jma2ycw+Vu/3Io2hjp9TB6c+o+43s1OH6z3JyKrjMXW0md0Zjqc7zeyv\nhus9yciq4zE1zcxuNLONZnbxcL0fGRnVjo+yZS4Kr99nZvv1tW6tx6Y0pj6DGFV8dEhbMXjnA9e5\n+y7A78PPPZhZBrgEOJZkpJXTzGz33tY3syzJ2NMfcPe9gDcC+Tq/F2kMdTmmgifdfb8wnVPPNyEN\npZ7HFMDXgGvq1HZpTPU6ph4ADnD3/YBjgG+G7cjoV69jaj1wgrvvQ1LP7Id1fRfSSOp1THUA/wz8\nQ32bLyOtj+OjtMxxwM7uvhT4APDtGtbt89hsdGY2bFOjGWgQo9GcCFwenl8OvLXCMgeTXDyudPc8\n8DPgpD7WPwa4390fAHD3V9w9rkP7pfHU65iSsatux5SZvRV4mqQmkIwddTmm3H1r6ruuDXjV3Yt1\naL80nnodU/e6+9ow/2GgzcxydWi/NJ56HVNb3P1moLNeDZeG0dvxUdJ9nLj77cBUM2vvY12dqzex\n0RLEmO3u68LzdcDsCsvMA1alfn4uzOtt/V0AN7PfmtldZvbxIW63NK56HVMAi0Oa9gozO3woGy0N\nrS7HlJlNBD4BfHaoGywNr26fU6FLyUPAQzRe9qXUTz2/+0pOBu4KFxQy+tX7mFIR/9Gvt+Ojr2Xm\n9rJuLcdmQxvLmRhVRycxs01U/2AYX5/mVGdm1wHtFV76VPoHd3czq9Tu8nlWYV75+lngcOBAYCvw\nezO7y91v6G/7pfGM0DH1PLDA3V8xs/2BX5nZnu6+sf/vQBrNCB1TnwW+7u5brBG/ZWRQRuiYwt3/\nDOxpZrsBvzWzFe7+ar/fgDSckTqmwr73JBll7uh+NVoa2kgeUzIm1Po3r+UcSMfWKFE1iOHuE4ez\nIX1x96pfeGa2zsza3X2tmc0BXqiw2GqSEVZK5od5ANXWXwXc5O4vh/38BtgfUBBjFBiJY8rduwjD\nFrv73Wb2FLAUuHvw70hG2gh9Th0MnGxmXwamArGZbXV3FY0dBUbomErv/9HwObUzcNeA34g0jJE6\npsxsPnAlcLq7PzPoNyINY6Q/p2TUKz8+FpBkVPS2zPywTK7C/FFzbI3le1ejpTvJVSSFogiPv6qw\nzJ3AUktGh2gBTg3r9bb+tcDeZtZmSZHPN5Kk1sroV5djysxmWCiQZ2Y7kQQwnq7LO5BGU5djyt3f\n4O6L3X0x8A3gXxXAGDPq9Tm1KHznYWYLST6nnqjLO5BGU69jaipJ4eFPuvutdWq7NKZ6naOXjN2r\nuLGjt+Oj5Crg7wDM7FBgQ+gqMphjSxqYuTd/5oyZTQOuAHYEVgLvcPcNZjYXuNTdjw/LvZnkJD8D\nfNfdv9jb+uG1dwH/SJJ6dI27N13lWum/eh1TZvZ24PMko9zEwKfdXSNKjAH1/JxK7eMzwEZ3/9qw\nvCkZUXX8nHo3SZX2fJg+7e6/Hc73JiOjjsfUP5McU+lg2NHu/uKwvDEZMXU+R18JTAJagFeAY9z9\n0WF7czJsKh0fZvZBAHf/TlimNArJZuBMd7+72rphfp/nVY3MzPyqq8pjOfVz4okn4u4NEzQcFUEM\nERERERERkbFgrAcxqtbEEBEREREREZHGo5oYIiIiIiIiIiINTkEMEREREREREWkK6k4iIiIiIiIi\n0kTUnUREREREREQkMLOfmdk9YXrGzO6pstxUM/uFmT1iZg+HYU4xs2VmdquZ3W9mV5nZpDB/mpnd\naGYbzezism39q5n9xcw21tjG3cI+OszsY4N9z9IclIkhIiIiIiIiPbj735aem9lXgWpDkF4I/Mbd\nTzGzLDAhzL8M+Ki7/9HMzgQ+Dnwa6AD+GdgrTGm/Bi6m55DMvXkJOA94a43LjxrKxBAREZHtmNmm\nOm//GjObbGZTzOzsAay/3Myu7uc6s8zsmiqvrTCzA/rbjnoxs1Yzu8nMdL4iIjJCLLlafgfw0wqv\nTQGOcPfvAbh7wd1fDS8vdfc/hufXAyeHZba4+81AZ/n23P3P7r62wn5mhmyPP4fpsLD8ene/E8gP\n/p1Ks9BJgYiISHVe1427H+/urwE7AOfUc18p5wI/qNYkhuA9hztxg+buncAfGYN32EREGsgRwDp3\nf6rCa4uB9Wb2fTO728wuNbPx4bWHzOyk8PxvgAVl6/bn++ZC4OvufjBwCkmWx5hmZsM2Vdn/sWb2\nqJk9YWafHM73riCGiIhIP5jZvmZ2m5ndZ2ZXmtnUMH+FmV1gZreb2WNmdniYP97MrjCzh8Lyt5nZ\n/uG1lWY2HbgAWBL6HX/ZzN6YzrAws0vM7Izw/NjQ7/gu4G2pZSaY2ffC/u82sxOrvIVTgGvCOm2h\nz/PDZnYl0Jba3jFmdouZ3RXaPyHMPy7s/04zu6jUTjP7rJn90Mz+BFxuZjMq3TWr1k4z2zPMuyf8\nbncOTbkKOG1QfzQREanIzK4zswcqTG9JLXYa8JMqm8gC+wPfcvf9gc3A+eG1s4BzzOxOYCLQNYim\nHgVcYkldjl8Dk1LBEhlmZpYBLgGOBfYATjOz3Ydr/6qJISIi0j//BXw49PH9HPAZ4O9J7ihl3P0Q\nM3tzmH80SYbFS+6+p5ntCdyb2lYp8+GTwJ7uvh8k3UTK9umAm9k44D+Bv3L3p8zs52y7k/Up4Pfu\nflYIrNxuZte7+5bSRsysHSim5p0NbHL3Pcxsb+DusNyMsL0j3X1ruMPyUTP7CvAfJKnDz5rZT+h5\nJ2034HB37wyvfd3dbzazHYHfkpzoVGwn8EHgQnf/iSWZHKVzlHuBw/r+s4iISH+5+9G9vR4+j99G\nEqio5DngOXe/I/z8C0IQw90fA/46bGcX4PhBNNWAQ9x9MIGQUSWKRjQf4WDgSXdfCUkRWOAk4JHh\n2LkyMURERGpkSd/fKak+vpcDb0gtcmV4vBtYFJ6/HvgZgLs/BNxfadO17J4kSPBMKqX3R6l1jwHO\nD3epbgRa2T51dyGwJvXzEWEbuPsDqbYdShJwuCVs7++AHYFdgafd/dmw3E9T+3fgqtAFBCrfNZtQ\npZ07ArcC/2RmnwAWuXtHaFcnEIUAjoiIDK+jgEfc/flKL4b6FatCkKK0/EOQ1LEIjxFJIc9vl63e\nn8qU1wIf6V7RbN9BbEsGbx6wKvXzc2HesFAmhoiIyMCVnzSVLuCL9PyO7e/JVYGeNxpKF/Dl/YfL\nt/t2d++ronv5OtV+vs7d39njBbNlfay7pey17e6ahb61ldr5qJndBpwA/MbMPujuN6a2Vdf6JCIi\nUtGplBX0NLO5wKXuXsqsOA/4sZm1AE8BZ4b5p5nZh8PzX7r7D1LbWAlMAlrM7K3A0e7+qJl9maT7\nSpuZrQr7+TxJAOObZnYfyffrH0i6qrQDdwCTgdjM/g+wh7vXtTB3I7A6jk5y3333cf/9le65dBvR\n72QFMURERGrk7q+a2Stmdri7/wk4HVjRx2o3k1R1X2FmewB7V1hmI8nJXMmzwB7hhHA8cCRJgctH\ngUVmtpO7P03PWhG/IznJOw/AzPZz93vK9vMs0J76+SbgncCNZrYXsA/JicltJCeLS0K3lQnAXOAx\nYCczWxiyMU5l24lM+dlU6a7ZV0N7lrn7fdXaaWaL3f0Z4OLQ/WTv0K5Wki4w21WxFxGR+nL3MyvM\ne55U15Dw2X5QheUuAi6qst1FVeZ/AvhEhfkvAX9bYf5ats86lEFatmwZy5Ztu2/xox/9qHyR1fT8\nvS8gycYYFupOIiIiUt14M1uVmv4vcAbwlXA3aB/g81XWLV3cfwuYaWYPAf9Ckmb7ao8Fk5Ozm0Mx\ntS+5+yrgCuBB4OeEWhXhQv4DwDWhsOe61H7+BciZ2f1m9iDwue0alJzsZUtFOklSeyea2cNh+TvD\nci8C7wF+Gt7nLcCuoYvHOcBvQ6G211LvpXxkk48AB4YinQ+R1LzorZ3vMLMHQzeTPUlqjwDsR9LV\nRERERBrDncBSM1sUbricSlKIe1iYu7IzRURE6iX0Bc6FYpdLgOuAXdy9MELt+SxJ/+afD3D9Ce6+\nOTz/JvC4u184hE0s39+/AXe4+//Uax8iIiLNxMz82muvHbb9HXPMMbh7j4zLUMT8G0AG+K67f3G4\n2qPuJCIiIvU1AbjBzHIkXS7OHqkARvBNkoKkAwpiAO+3ZLjXFpIMke8MVcPKha4kh5OMaCIiIiIN\nwt3/F/jfkdi3MjFEREREREREmoSZ+XXXXTds+zv66KO3y8QYSaqJISIiIiIiIiJNQd1JRERERERE\nRJpIFI3dfISx+85FREREREREpKkoE0NERERERESkiZg1TImKYadMDBERERERERFpCgpiiIiIiIiI\niEhTUBBDRERERERERJqCamKIiIiIiIiINBHVxBARERERERERaXAKYoiIiIiIiIhIU1B3EhERERER\nEZEmou4kIiIiIiIiIiINTpkYIiIiIiIiIk1EmRgiIiIiIiIiIg1OmRgiIiIiIiIiTUSZGCIiIiIi\nIiIiDU6ZGCIiIiIiIiJNRJkYIiIiIiIiIiINTpkYIiIiIiIiIk1EmRgiIiIiIiIiIg1OmRgiIiIi\nIiIiTUSZGCIiIiIiIiIiDU5BDBERERERERFpCgpiiIiIiIiIiEhTUBBDRERERERERJqCCnuKiIiI\niIiINBEV9hQRERERERERaXDKxBARERERERFpIsrEEBERERERERFpcMrEEBEREREREWkiysQQERER\nEREREWlwysQQERERERERaSLKxBARERERERERaXAKYoiIiIiIiIhIU1B3EhEREREREZEmou4kIiIi\nIiIiIiINTpkYIiIiIiIiIk1EmRgiIiIiIiIiIg1OmRgiIiIiIiIiTUSZGCIiIiIiIiIiDU5BDBER\nERERERFpCgpiiIiIiIiIiEhTUE0MERERERERkSaimhgiIiIiIiIiIg1OmRgiIiIiIiIiTUSZGCIi\nIiIiIiIiDU5BDBERERERERFpCupOIiIiIiIiItJE1J1ERERERERERKTBKYghIiIiIiIi0kTMbNim\nAbTtPDN7xMweNLMvpeb/o5k9YWaPmtkxA33v6k4iIiIiIiIiIoNmZn8FnAjs4+55M5sZ5u8BnArs\nAcwDrjezXdw97u8+FMQQERERERERaSINXBPjbOCL7p4HcPf1Yf5JwE/D/JVm9iRwMHBbf3eg7iQi\nIiIiIiIiMhSWAm8ws9vMbIWZHRjmzwWeSy33HElGRr8pE0NERERERESkiYxkJoaZXQe0V3jpUyQx\nhh3c/VAzOwi4AtipyqZ8IPtXEENEREREREREALjjjju44447qr7u7kdXe83MzgauDMvdYWaxmc0A\nVgMLUovOD/P6zdwHFPwQERERERERkWFmZv7AAw8M2/723ntv3L2m1A8z+yAw190/Y2a7ANe7+46h\nsOdPSOpgzAOuB3b2AQQklIkhIiIiIiIiIkPhe8D3zOwBoAv4OwB3f9jMrgAeBgrAOQMJYIAyMURE\nRERERESaRiNnYgwHZWKIiIiIiIiINJEGHmK17jTEqoiIiIiIiIg0BQUxRERERERERKQpKIghIiIi\nIiIiIk1BNTFEREREREREmohqYoiIiIiIiIiINDhlYoiIiIiIiIg0EWViiIiIiIiIiIg0OGViiIiI\niIiIiDQRZWKIiIiIiIiIiDQ4ZWKIiIiIiIiINBFlYoiIiIiIiIiINDgFMURERERERESkKag7iYiI\niIiIiEgTUXcSEREREREREZEGp0wMERERERERkSaiTAwRERERERERkQanIIaIiIiIiIiINAUFMURE\nRERERESkKagmhoiIiIiIiEgTUU0MEREREREREZEGp0wMERERERERkSaiTAwRERERERERkQanIIaI\niIiIiIiINAUFMURERERERESkKSiIISIiIiIiIiJNQYU9RURERERERJqICnuKiIiIiIiIiDQ4ZWKI\niIiIiIiINBFlYoiIiIiIiIiINDhlYoiIiIiIiIg0EWViiIiIiIiIiIg0OAUxRERERERERKQpKIgh\nIiIiIiIiIk1BNTFEREREREREmohqYoiIiIiIiIiINDgFMURERERERESkKag7iYiIiIiIiEgTUXcS\nEREREREREZEGpyCGiIiIiIiIiDQFBTFEREREREREpCmoJoaIiIiIiIhIE1FNDBERERERERGRBqdM\nDBEREREREZEmokwMEREREREREZEGp0wMERERERERkSaiTAwRERERERERkQanIIaIiIiIiIiINAUF\nMURERERERESkKSiIISIiIiIiIiJNQUEMERERERERkSZiZsM29bNdf2NmD5lZ0cwOSM0/2szuNLP7\nw+NfpV47wMweMLMnzOzCvvahIIaIiIiIiIiIDIUHgLcBNwGemr8eOMHd9wHOAH6Yeu3bwHvdfSmw\n1MyO7W0HGmJVREREREREpIk06hCr7v4obN8+d7839ePDQJuZ5YAZwCR3/3N47b+AtwK/rbYPZWKI\niIiIiIiIyHA5GbjL3fPAPOC51Gurw7yqlIkhIiIiIiIi0kRGMhPDzK4D2iu89E/ufnUf6+4JXAAc\nPdD9K4ghIiIiIiIiIgDccsst3HLLLVVfd/cBBSDMbD5wJXC6uz8TZq8G5qcWmx/mVd+Ou/f2uoiI\niIiIiIg0CDPzNWvWDNv+5syZg7v3K/XDzG4E/sHd7wo/TwX+AHzG3X9VtuztwEeAPwPXABe5u2pi\niIiIiIiIiEj9mNnbzGwVcChwjZn9b3jpXGAJ8BkzuydMM8Jr5wCXAU8AT/YWwABlYoiIiIiIiIg0\nDTPztWvXDtv+2tvb+52JUU/KxBARERERERGRpqAghoiIiIiIiIg0BY1OIiIiIiIiItJERnKI1ZGm\nTAwRERERERERaQoKYoiIiIiIiIhIU1AQQ0RERERERESagmpiiIiIiIiIiDQR1cQQEREREREREWlw\nysQQERERERERaSLKxBARERERERERaXDKxBARERERERFpIsrEEBERERERERFpcApiiIiIiIiIiEhT\nUBBDRERERERERJqCghgiIiIiIiIi0hRU2FNERERERESkiaiwp4iIiIiIiIhIg1MmhoiIiIiIiEgT\nUSaGiIiIiIiIiEiDUxBDRERERERERJqCghgiIiIiIiIi0hRUE0NERERERESkiagmhoiIiIiIiIhI\ng1MmhoiIiIiIiEgTUSaGiIiIiIiIiEiDUxBDRERERERERJqCghgiIiIiIiIi0hQUxBARERERERGR\npqDCniIiIiIi8v+3d8csctVRGIffgyBZFxsrg2BlChsLt9BOSJXSFIIBU1koVoKN+C1sFEEtQsA2\nIBJFJDAGixQpXJXF2KloKTYLghyLHXBZkglm13gPPk8385+5d+ofe98FBjHsCQAAALBwIgYAAAAw\ngogBAAAAjGATAwAAAAaxiQEAAACwcCIGAAAAMIKIAQAAAIxgEwMAAAAGsYkBAAAAsHAiBgAAADCC\niAEAAACMIGIAAAAAIxj2BAAAgEEMewIAAAAsnIgBAAAAjCBiAAAAACPYxAAAAIBBbGIAAAAALJyI\nAQAAAIwgYgAAAAAjiBgAAADACIY9AQAAYBDDngAAAADHUFUvVNW3VfVnVT196P1TVfVRVX1dVd9V\n1ZuHznaqareqblXV23e7h4gBAAAAnITdJOeTrI68/2KSdPdTSXaSvFJVj6/P3k3ycncPwGBKAAAC\naklEQVSfSXKmqs5tuoGIAQAAABxbd+919/e3OfolyXZVPZBkO8kfSX6vqtNJHu7uG+vPXUry/KZ7\n2MQAAACAQaZtYnT3Z1V1MQcx46Ekr3f3b1X1RJKfDn305ySPbbqWiAEAAAAkSVarVVaro0+D/K2q\nPk/y6G2O3uruj+/wnZeSbCU5neSRJF9W1Rf38vuqu+/lewAAAMB9VlW9v79/3+63tbWV7v5Hf/pR\nVdeSvNHdN9ev30nyVXdfXr/+IMnVJNeTXOvuJ9fvX0jyXHe/eqdr28QAAAAATtrh8LGX5GySVNV2\nkmeT7HX3rznYxnimDp6RuZjkyqaLihgAAADAsVXV+ar6MQeR4pOquro+ei/Jg1W1m+RGkg+7+5v1\n2WtJ3k9yK8kP3f3pxnt4nAQAAABmmPA4yb/JsCcAAAAMMu2/k5wkj5MAAAAAI4gYAAAAwAgiBgAA\nADCCiAEAAACMYNgTAAAABjHsCQAAALBwIgYAAAAwgogBAAAAjGATAwAAAAaxiQEAAACwcCIGAAAA\nMIKIAQAAAIwgYgAAAAAjGPYEAACAQQx7AgAAACyciAEAAACMIGIAAAAAI9jEAAAAgEFsYgAAAAAs\nnIgBAAAAjCBiAAAAACPYxAAAAIBBbGIAAAAALJyIAQAAAIwgYgAAAAAjiBgAAADACIY9AQAAYBDD\nngAAAAALJ2IAAAAAI4gYAAAAwAg2MQAAAGAQmxgAAAAACydiAAAAACOIGAAAAMAINjEAAABgkP/z\nJkZ193/9GwAAAADuyuMkAAAAwAgiBgAAADCCiAEAAACMIGIAAAAAI4gYAAAAwAh/AQgaAJlwbvDi\nAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "## Create a 2D surface plot\n", "\n", @@ -213,10 +173,12 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": false }, "outputs": [], - "source": [] + "source": [ + "pose.index.values" + ] } ], "metadata": { From 09aaca586098ff6f4d76449e74b909fa33b02c3e Mon Sep 17 00:00:00 2001 From: Pras Velagapudi Date: Sun, 27 Nov 2016 13:37:55 -0500 Subject: [PATCH 06/43] Added a data export script that writes CSV for a given logfile. --- notebooks/Data_Exporter.ipynb | 97 +++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 notebooks/Data_Exporter.ipynb diff --git a/notebooks/Data_Exporter.ipynb b/notebooks/Data_Exporter.ipynb new file mode 100644 index 0000000..a9b38c5 --- /dev/null +++ b/notebooks/Data_Exporter.ipynb @@ -0,0 +1,97 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas\n", + "import platypus.io.logs\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Import the data from the specified logfile\n", + "log_filename = '../logs/platypus_20161126_'\n", + "data = platypus.io.logs.load(log_filename)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Extract position of boat and remove duplicate entries.\n", + "data[\"pose\"][\"time\"] = data[\"pose\"].index\n", + "position = data['pose'][['time', 'latitude', 'longitude']].drop_duplicates(subset='time', keep='first')\n", + "position = position[['latitude', 'longitude']] # remove unneeded time column after de-duplication\n", + "\n", + "# Put together all the sensor data we are looking for.\n", + "# This also combines the name of the sensor with the name of each channel.\n", + "sensor_names = ['ATLAS_DO', 'ES2', 'HDS_TEMP', 'HDS_DEPTH']\n", + "sensor_frames = []\n", + "for sensor_name in sensor_names:\n", + " sensor_data = data[sensor_name].copy()\n", + " sensor_data.columns = [ sensor_name + '_' + str(k) for k in sensor_data.columns ]\n", + " sensor_frames.append(sensor_data)\n", + "sensor_data = pandas.concat(sensor_frames, axis=1)\n", + "\n", + "# Find the position for each sensor reading.\n", + "sensor_position = position.reindex(sensor_data.index, method='nearest')\n", + "output = pandas.concat((sensor_position, sensor_data), axis=1)\n", + "\n", + "# Fill in missing values with last known values.\n", + "output = output.apply(pandas.Series.interpolate, method='nearest')\n", + "output['epoch_time'] = output.index.astype(np.int64)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Output the file as a CSV in the same directory as the original logfile.\n", + "output_filename = os.path.splitext(log_filename)[0] + '.csv'\n", + "output.to_csv(output_filename, index=True)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.9" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From 0dabae6352584c30dcef7df08e31f1592acf4e4e Mon Sep 17 00:00:00 2001 From: Pras Velagapudi Date: Sun, 27 Nov 2016 13:47:57 -0500 Subject: [PATCH 07/43] Added data export function cleanup. This automates the detections of sensor channels in the logfile. --- notebooks/Data_Exporter.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/Data_Exporter.ipynb b/notebooks/Data_Exporter.ipynb index a9b38c5..55a3491 100644 --- a/notebooks/Data_Exporter.ipynb +++ b/notebooks/Data_Exporter.ipynb @@ -23,7 +23,7 @@ "outputs": [], "source": [ "# Import the data from the specified logfile\n", - "log_filename = '../logs/platypus_20161126_'\n", + "log_filename = '../logs/platypus_20161126_150201.txt'\n", "data = platypus.io.logs.load(log_filename)" ] }, @@ -42,7 +42,7 @@ "\n", "# Put together all the sensor data we are looking for.\n", "# This also combines the name of the sensor with the name of each channel.\n", - "sensor_names = ['ATLAS_DO', 'ES2', 'HDS_TEMP', 'HDS_DEPTH']\n", + "sensor_names = [k for k in data.keys() if k not in ['pose', 'BATTERY']]\n", "sensor_frames = []\n", "for sensor_name in sensor_names:\n", " sensor_data = data[sensor_name].copy()\n", From e8820ca4d743ad1102ab6a7bb21afcad30b5014d Mon Sep 17 00:00:00 2001 From: Pras Velagapudi Date: Sun, 27 Nov 2016 14:02:14 -0500 Subject: [PATCH 08/43] Added switch for partial sensor export. --- notebooks/Data_Exporter.ipynb | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/notebooks/Data_Exporter.ipynb b/notebooks/Data_Exporter.ipynb index 55a3491..d658128 100644 --- a/notebooks/Data_Exporter.ipynb +++ b/notebooks/Data_Exporter.ipynb @@ -24,7 +24,12 @@ "source": [ "# Import the data from the specified logfile\n", "log_filename = '../logs/platypus_20161126_150201.txt'\n", - "data = platypus.io.logs.load(log_filename)" + "data = platypus.io.logs.load(log_filename)\n", + "\n", + "# To selectively export certain sensors, change this value to an array of names.\n", + "# By default, all sensors are exported (meaning they are also interpolated).\n", + "sensor_names = None\n", + "# sensor_names = ['ATLAS_DO', 'ES2']" ] }, { @@ -40,9 +45,12 @@ "position = data['pose'][['time', 'latitude', 'longitude']].drop_duplicates(subset='time', keep='first')\n", "position = position[['latitude', 'longitude']] # remove unneeded time column after de-duplication\n", "\n", + "# If no sensor is specified, try to export all detected sensors.\n", + "if not sensor_names:\n", + " sensor_names = [k for k in data.keys() if k not in ['pose', 'BATTERY']]\n", + "\n", "# Put together all the sensor data we are looking for.\n", - "# This also combines the name of the sensor with the name of each channel.\n", - "sensor_names = [k for k in data.keys() if k not in ['pose', 'BATTERY']]\n", + "# This also combines the name of the sensor with the name of each channel. \n", "sensor_frames = []\n", "for sensor_name in sensor_names:\n", " sensor_data = data[sensor_name].copy()\n", From 7187f2cd60493001a83661587a8159ad932f1726 Mon Sep 17 00:00:00 2001 From: Pras Velagapudi Date: Sun, 27 Nov 2016 14:18:20 -0500 Subject: [PATCH 09/43] Added combined and independent data export scripts. --- ...ter.ipynb => Data_Exporter_Combined.ipynb} | 12 +- notebooks/Data_Exporter_Indepedent.ipynb | 126 ++++++++++++++++++ 2 files changed, 137 insertions(+), 1 deletion(-) rename notebooks/{Data_Exporter.ipynb => Data_Exporter_Combined.ipynb} (82%) create mode 100644 notebooks/Data_Exporter_Indepedent.ipynb diff --git a/notebooks/Data_Exporter.ipynb b/notebooks/Data_Exporter_Combined.ipynb similarity index 82% rename from notebooks/Data_Exporter.ipynb rename to notebooks/Data_Exporter_Combined.ipynb index d658128..663b9d3 100644 --- a/notebooks/Data_Exporter.ipynb +++ b/notebooks/Data_Exporter_Combined.ipynb @@ -1,5 +1,15 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Log Data Exporter - Combined Sensor CSV\n", + "This data exporter takes a log file and exports it to a single CSV files, with data values for each sensor.\n", + "\n", + "Since each sensor is exported together, these CSV files will have interpolated data values. This is because each row represents an update in _one_ sensor value, but there must be valid entries for the other sensors at the time. The values of all of the other sensors will be interpolated to the nearest valid reading (in time) for these events." + ] + }, { "cell_type": "code", "execution_count": null, @@ -23,7 +33,7 @@ "outputs": [], "source": [ "# Import the data from the specified logfile\n", - "log_filename = '../logs/platypus_20161126_150201.txt'\n", + "log_filename = '../logs/platypus_20161126_161922.txt'\n", "data = platypus.io.logs.load(log_filename)\n", "\n", "# To selectively export certain sensors, change this value to an array of names.\n", diff --git a/notebooks/Data_Exporter_Indepedent.ipynb b/notebooks/Data_Exporter_Indepedent.ipynb new file mode 100644 index 0000000..ac18813 --- /dev/null +++ b/notebooks/Data_Exporter_Indepedent.ipynb @@ -0,0 +1,126 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Log Data Exporter - Independent Sensor CSVs\n", + "This data exporter takes a log file and exports it to a series of CSV files, one per sensor.\n", + "\n", + "Since each sensor is exported separately, these CSV files will not have interpolated data values, only the raw reported sensor values." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas\n", + "import platypus.io.logs\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Import the data from the specified logfile\n", + "log_filename = '../logs/platypus_20161126_161922.txt'\n", + "data = platypus.io.logs.load(log_filename)\n", + "\n", + "# To selectively export certain sensors, change this value to an array of names.\n", + "# By default, all sensors are exported (meaning they are also interpolated).\n", + "sensor_names = None\n", + "# sensor_names = ['ATLAS_DO', 'ES2']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "def export_csv(data, sensor_name, output_filename):\n", + " \"\"\"\n", + " Export a CSV for a particular sensor in a logfile.\n", + "\n", + " :param data: a loaded Platypus data log\n", + " :param sensor_name: the name of a sensor in the data log\n", + " :param output_file: the filename to use for the output CSV\n", + " \"\"\"\n", + " # Extract position of boat and remove duplicate entries.\n", + " data[\"pose\"][\"time\"] = data[\"pose\"].index\n", + " position = data['pose'][['time', 'latitude', 'longitude']].drop_duplicates(subset='time', keep='first')\n", + " position = position[['latitude', 'longitude']] # remove unneeded time column after de-duplication\n", + "\n", + " # Check if the sensor name exists in the data log.\n", + " if sensor_name not in data.keys():\n", + " raise ValueError(\"'{:s}' sensor was not found in the log file.\"\n", + " .format(sensor_name))\n", + "\n", + " # Put together all the sensor data we are looking for.\n", + " sensor_data = data[sensor_name].copy()\n", + "\n", + " # Find the position for each sensor reading.\n", + " sensor_position = position.reindex(sensor_data.index, method='nearest')\n", + " output = pandas.concat((sensor_position, sensor_data), axis=1)\n", + "\n", + " # Fill in missing values with last known values.\n", + " output = output.apply(pandas.Series.interpolate, method='nearest')\n", + " output['epoch_time'] = output.index.astype(np.int64)\n", + " \n", + " # Output the file as a CSV in the same directory as the original logfile.\n", + " output.to_csv(output_filename, index=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# If no sensor is specified, try to export all detected sensors.\n", + "if not sensor_names:\n", + " sensor_names = [k for k in data.keys() if k not in ['pose', 'BATTERY']]\n", + "\n", + "# Iterate through and export each sensor as the original logfile name + sensor_name\n", + "for sensor_name in sensor_names:\n", + " output_filename = os.path.splitext(log_filename)[0] + '_' + str(sensor_name) + '.csv'\n", + " export_csv(data, sensor_name, output_filename)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.9" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From 0a83a1c25c79e25f6fc025089e6ccde21db9ca25 Mon Sep 17 00:00:00 2001 From: Pras Velagapudi Date: Sun, 27 Nov 2016 14:19:12 -0500 Subject: [PATCH 10/43] Fixed typo in spelling. --- ..._Exporter_Indepedent.ipynb => Data_Exporter_Independent.ipynb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename notebooks/{Data_Exporter_Indepedent.ipynb => Data_Exporter_Independent.ipynb} (100%) diff --git a/notebooks/Data_Exporter_Indepedent.ipynb b/notebooks/Data_Exporter_Independent.ipynb similarity index 100% rename from notebooks/Data_Exporter_Indepedent.ipynb rename to notebooks/Data_Exporter_Independent.ipynb From 200a39ac4fd38668b312f74f4ad5c8e75801666d Mon Sep 17 00:00:00 2001 From: Pras Velagapudi Date: Sun, 27 Nov 2016 17:27:28 -0500 Subject: [PATCH 11/43] Minor bugfixes to loading. --- setup.py | 1 + src/platypus/util/conversions.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8d8033a..50da608 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ 'pymongo', 'pyserial', 'six', + 'scipy', 'utm' ], test_suite="tests", diff --git a/src/platypus/util/conversions.py b/src/platypus/util/conversions.py index 8a61cbd..762c996 100644 --- a/src/platypus/util/conversions.py +++ b/src/platypus/util/conversions.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Module containing utility conversion functions. Copyright 2015. Platypus LLC. All rights reserved. From 2b798a5a2ae50288b99979880c29f8aaae6e0952 Mon Sep 17 00:00:00 2001 From: Pras Velagapudi Date: Sun, 27 Nov 2016 17:28:42 -0500 Subject: [PATCH 12/43] [WIP] Added updates to render actual map overlay. --- notebooks/Data_Interpolation.ipynb | 161 +++++++++++++++-------------- 1 file changed, 85 insertions(+), 76 deletions(-) diff --git a/notebooks/Data_Interpolation.ipynb b/notebooks/Data_Interpolation.ipynb index c5d1782..95da9ef 100644 --- a/notebooks/Data_Interpolation.ipynb +++ b/notebooks/Data_Interpolation.ipynb @@ -8,14 +8,11 @@ }, "outputs": [], "source": [ - "%matplotlib inline\n", - "\n", "import numpy as np\n", "import numpy.lib.recfunctions\n", "import scipy\n", "import scipy.interpolate\n", "import pandas\n", - "import matplotlib.pyplot as plt\n", "import platypus.io.logs" ] }, @@ -23,23 +20,18 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": false }, "outputs": [], "source": [ "# Import the data from the specified logfile\n", "log_filename = '../logs/platypus_20161126_150201.txt'\n", - "data = platypus.io.logs.load(log_filename)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ + "data = platypus.io.logs.load(log_filename)\n", + "\n", + "# Define useful access variables.\n", + "pose = data['pose']\n", + "position = pose[['latitude', 'longitude']]\n", + "\n", "# Print the available sensors and channels for this logfile.\n", "print \"Available sensors/channels:\"\n", "for s in data.keys():\n", @@ -70,8 +62,7 @@ }, "outputs": [], "source": [ - "# Extract the pose and the sensor data of interest.\n", - "pose = data['pose']\n", + "# Extract the pose timing and the sensor data of interest.\n", "pose_times = pose.index.values.astype(np.float64)\n", "\n", "sensor = data[sensor_name]\n", @@ -90,6 +81,15 @@ "sensor = sensor[sensor_valid]" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, @@ -98,36 +98,14 @@ }, "outputs": [], "source": [ - "## Create a 2D surface plot\n", - "\n", - "# Create matrices of the positions and data values.\n", - "points = sensor[['latitude', 'longitude']].as_matrix()\n", - "values = sensor[sensor_data]\n", - "\n", - "# Create a grid over the range of the data.\n", - "grid_x, grid_y = np.mgrid[sensor.latitude.max():sensor.latitude.min():1000j,\n", - " sensor.longitude.min():sensor.longitude.max():1000j]\n", - "grid_z = scipy.interpolate.griddata(points, values, (grid_x, grid_y), method='cubic')\n", - "\n", - "# Plot the sensor data as an image.\n", - "fig = plt.figure(figsize=(20,10))\n", - "plt.imshow(grid_z, aspect='equal', cmap='Greys',\n", - " extent=[sensor.longitude.min(), sensor.longitude.max(),\n", - " sensor.latitude.min(), sensor.latitude.max()])\n", - "\n", - "plt.colorbar()\n", - "plt.title(\"Sensor: \" + sensor_name + \"\\n\")\n", - "plt.xlabel('Longitude (degrees)')\n", - "plt.ylabel('Latitude (degrees)')\n", - "plt.hold(True)\n", - "\n", - "# Plot the trajectory of the vehicle as an overlay.\n", - "plt.plot(points[:,1], points[:,0], 'c-', linewidth=3)\n", - "plt.plot(points[0,1], points[0,0], 'co')\n", - "plt.show()\n", - "\n", - "# Save the figure to disk.\n", - "fig.savefig(sensor_name + '.png', dpi=fig.dpi)" + "from ipyleaflet import (\n", + " Map,\n", + " Marker,\n", + " TileLayer, ImageOverlay,\n", + " Polyline, Polygon, Rectangle, Circle, CircleMarker,\n", + " GeoJSON,\n", + " DrawControl\n", + ")" ] }, { @@ -138,35 +116,45 @@ }, "outputs": [], "source": [ - "## Create a 3D trail plot\n", - "x = sensor['longitude'].as_matrix()\n", - "y = sensor['latitude'].as_matrix()\n", - "z = sensor[sensor_data].as_matrix()\n", - "\n", - "import matplotlib.cm as cm\n", - "import matplotlib.colors as colors\n", - "\n", - "# Create normalized color entries for each sensor reading.\n", - "norm = colors.Normalize(min(z), max(z))\n", - "colors = cm.jet(norm(z))\n", - "m = cm.ScalarMappable(cmap=cm.jet, norm=norm)\n", - "m.set_array(colors)\n", - "\n", - "# Create a 3D plot that creates a vertical trail.\n", - "from mpl_toolkits.mplot3d import Axes3D\n", - "fig = plt.figure(figsize=(20,10))\n", - "ax = fig.add_subplot(111, projection='3d')\n", - "ax.bar3d(x, y, [min(z)] * len(z), 5e-5, 5e-5, z - min(z), color=colors, alpha=0.8, edgecolor='none')\n", - "\n", - "plt.xlabel('Longitude (degrees)')\n", - "plt.ylabel('Latitude (degrees)')\n", - "plt.title('Sensor: ' + sensor_name, fontsize=20)\n", + "# Create a map centered on this data log.\n", + "center = [pose['latitude'].median(), pose['longitude'].median()]\n", + "zoom = 17\n", + "m = Map(center=center, zoom=zoom)\n", + "m" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Add trail for the vehicle to map\n", + "pl = Polyline(locations=position.as_matrix().tolist())\n", + "pl.fill_opacity = 0.0\n", + "pl.weight = 2\n", + "m += pl" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Add a data overlay for the map\n", + "data_padding = 0.0001 # degrees lat/lon\n", + "data_bounds = [(position.min() - data_padding).tolist(),\n", + " (position.max() + data_padding).tolist()]\n", "\n", - "fig.colorbar(m, shrink=0.5, aspect=5)\n", - "fig.show()\n", + "data_grid = xv, yv = meshgrid(x, y, sparse=False, indexing='ij')\n", "\n", - "# Save the figure to disk.\n", - "fig.savefig(sensor_name + '_3d.png', dpi=fig.dpi)" + "io = ImageOverlay(url='http://ipython.org/_static/IPy_header.png', bounds=data_bounds)\n", + "m += io" ] }, { @@ -177,8 +165,17 @@ }, "outputs": [], "source": [ - "pose.index.values" + "m.bounds" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] } ], "metadata": { @@ -197,7 +194,19 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", - "version": "2.7.9" + "version": "2.7.12" + }, + "widgets": { + "state": { + "48b6b0a628a84124a497bdef12f2ff15": { + "views": [ + { + "cell_index": 6 + } + ] + } + }, + "version": "1.2.0" } }, "nbformat": 4, From a9995ae0b68a296cb7aec37f60a9ee5b6c16bc4b Mon Sep 17 00:00:00 2001 From: Pras Velagapudi Date: Mon, 28 Nov 2016 13:28:39 -0500 Subject: [PATCH 13/43] Created very barebones rendering system for logs. --- notebooks/Data_Interpolation.ipynb | 143 ++++++++++++++++------------- 1 file changed, 78 insertions(+), 65 deletions(-) diff --git a/notebooks/Data_Interpolation.ipynb b/notebooks/Data_Interpolation.ipynb index 95da9ef..02b8bbc 100644 --- a/notebooks/Data_Interpolation.ipynb +++ b/notebooks/Data_Interpolation.ipynb @@ -8,12 +8,21 @@ }, "outputs": [], "source": [ + "%matplotlib inline\n", + "\n", + "from ipyleaflet import Map, ImageOverlay, Polyline\n", + "import matplotlib\n", + "import matplotlib.cm\n", + "from matplotlib import pyplot\n", "import numpy as np\n", "import numpy.lib.recfunctions\n", "import scipy\n", "import scipy.interpolate\n", "import pandas\n", - "import platypus.io.logs" + "import platypus.io.logs\n", + "import os\n", + "import uuid\n", + "import glob" ] }, { @@ -25,7 +34,7 @@ "outputs": [], "source": [ "# Import the data from the specified logfile\n", - "log_filename = '../logs/platypus_20161126_150201.txt'\n", + "log_filename = '../logs/platypus_20161126_161922.txt'\n", "data = platypus.io.logs.load(log_filename)\n", "\n", "# Define useful access variables.\n", @@ -50,8 +59,9 @@ "outputs": [], "source": [ "# Select the sensor and the name of the channel for that sensor.\n", - "sensor_name = 'ATLAS_DO'\n", - "sensor_data = 'do'" + "sensor_name = 'HDS_DEPTH'\n", + "sensor_channel = 1\n", + "sensor_units = 'meters (m)'" ] }, { @@ -81,33 +91,6 @@ "sensor = sensor[sensor_valid]" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "from ipyleaflet import (\n", - " Map,\n", - " Marker,\n", - " TileLayer, ImageOverlay,\n", - " Polyline, Polygon, Rectangle, Circle, CircleMarker,\n", - " GeoJSON,\n", - " DrawControl\n", - ")" - ] - }, { "cell_type": "code", "execution_count": null, @@ -116,26 +99,10 @@ }, "outputs": [], "source": [ - "# Create a map centered on this data log.\n", - "center = [pose['latitude'].median(), pose['longitude'].median()]\n", - "zoom = 17\n", - "m = Map(center=center, zoom=zoom)\n", - "m" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# Add trail for the vehicle to map\n", + "# Create a trail of the vehicle's path on the map.\n", "pl = Polyline(locations=position.as_matrix().tolist())\n", "pl.fill_opacity = 0.0\n", - "pl.weight = 2\n", - "m += pl" + "pl.weight = 2" ] }, { @@ -147,14 +114,48 @@ "outputs": [], "source": [ "# Add a data overlay for the map\n", - "data_padding = 0.0001 # degrees lat/lon\n", + "data_padding = [0.0001, 0.0001] # degrees lat/lon\n", + "data_resolution = [0.00001, 0.00001] # degrees lat/lon\n", + "data_interpolation_radius = 0.00005 # degrees lat/lon\n", "data_bounds = [(position.min() - data_padding).tolist(),\n", " (position.max() + data_padding).tolist()]\n", "\n", - "data_grid = xv, yv = meshgrid(x, y, sparse=False, indexing='ij')\n", + "# Create a rectangular grid of overlay points.\n", + "data_xv, data_yv = np.meshgrid(\n", + " np.arange(data_bounds[1][0], data_bounds[0][0], -data_resolution[0]),\n", + " np.arange(data_bounds[0][1], data_bounds[1][1], data_resolution[1])\n", + ")\n", + "data_shape = data_xv.shape\n", + "data_xy = np.vstack([data_xv.ravel(), data_yv.ravel()]).T\n", + "\n", + "# Create a radial-basis interpolator over the sensor dataset\n", + "# Then, query it at each point of the rectangular grid.\n", + "from sklearn.neighbors import RadiusNeighborsClassifier\n", + "data_estimator = RadiusNeighborsClassifier(radius=data_interpolation_radius, outlier_label=np.nan)\n", + "data_estimator.fit(sensor[['latitude','longitude']], sensor[sensor_channel])\n", + "data_zv = data_estimator.predict(data_xy)\n", + "data_zv = data_zv.reshape(data_shape).T\n", "\n", - "io = ImageOverlay(url='http://ipython.org/_static/IPy_header.png', bounds=data_bounds)\n", - "m += io" + "# Normalize data from [0, 1)\n", + "data_max = data_zv[np.isfinite(data_zv)].max()\n", + "data_min = data_zv[np.isfinite(data_zv)].min()\n", + "data_zv = (data_zv - data_min) / (data_max - data_min)\n", + "\n", + "# Update a color map only at the points that have valid values.\n", + "data_rgb = np.zeros((data_shape[0], data_shape[1], 4), dtype=np.uint8)\n", + "data_rgb = matplotlib.cm.jet(data_zv)*255.0\n", + "data_rgb[:,:,3] = 255 * np.isfinite(data_zv)\n", + "\n", + "# Remove any old image files.\n", + "old_png_files = glob.glob('./*.png')\n", + "for old_png_file in old_png_files:\n", + " os.remove(old_png_file)\n", + "\n", + "png_filename = './platypus_data_{:s}.png'.format(uuid.uuid4())\n", + "scipy.misc.imsave(png_filename, data_rgb)\n", + "\n", + "# Create image overlay that references generated image.\n", + "io = ImageOverlay(url=png_filename, bounds=data_bounds)" ] }, { @@ -165,17 +166,29 @@ }, "outputs": [], "source": [ - "m.bounds" + "# Create a map centered on this data log.\n", + "center = [pose['latitude'].median(), pose['longitude'].median()]\n", + "zoom = 18\n", + "m = Map(center=center, zoom=zoom, height='600px')\n", + "m += io # Add image overlay\n", + "m += pl # Add vehicle trail\n", + "\n", + "# Make a figure and axes with dimensions as desired.\n", + "fig = pyplot.figure(figsize=(15, 3))\n", + "ax1 = fig.add_axes([0.05, 0.80, 0.9, 0.15])\n", + "\n", + "# Set the colormap and norm to correspond to the data for which\n", + "# the colorbar will be used.\n", + "cmap = matplotlib.cm.jet\n", + "norm = matplotlib.colors.Normalize(vmin=data_min, vmax=data_max)\n", + "cb1 = matplotlib.colorbar.ColorbarBase(ax1, cmap=cmap,\n", + " norm=norm,\n", + " orientation='horizontal')\n", + "cb1.set_label(sensor_units)\n", + "\n", + "pyplot.show()\n", + "m" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] } ], "metadata": { @@ -194,11 +207,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", - "version": "2.7.12" + "version": "2.7.9" }, "widgets": { "state": { - "48b6b0a628a84124a497bdef12f2ff15": { + "353ccbbc0ff94b5bbe0afcc012b3fd5c": { "views": [ { "cell_index": 6 From 567a692b7a13e1c56ef44cfd7d96f9ee202d21ad Mon Sep 17 00:00:00 2001 From: Pras Velagapudi Date: Tue, 29 Nov 2016 10:11:53 -0500 Subject: [PATCH 14/43] Update README.md --- notebooks/README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/notebooks/README.md b/notebooks/README.md index f15b7ef..c1785d6 100644 --- a/notebooks/README.md +++ b/notebooks/README.md @@ -1,3 +1,17 @@ # Platypus Analytics Notebooks -This directory contains example notebooks that demonstrate various useful data analysis operations. +This directory contains example IPython/Jupyter notebooks that demonstrate various useful data analysis operations. + +# Usage + +If you do not already have it installed, install [Jupyter](https://jupyter.readthedocs.io/en/latest/install.html#alternative-for-experienced-python-users-installing-jupyter-with-pip): +``` +$ pip install jupyter +``` + +Then, simply browse to this directory and run: +``` +$ jupyter notebook +``` + +This should create a Jupyter instance showing the notebooks in this directory. From a7800d0c3aa949168eac9577cc4ecba172034c93 Mon Sep 17 00:00:00 2001 From: Pras Velagapudi Date: Thu, 1 Dec 2016 20:05:52 -0500 Subject: [PATCH 15/43] Added missing float cast (for integer values). --- notebooks/Data_Interpolation.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/Data_Interpolation.ipynb b/notebooks/Data_Interpolation.ipynb index 02b8bbc..89ef43c 100644 --- a/notebooks/Data_Interpolation.ipynb +++ b/notebooks/Data_Interpolation.ipynb @@ -132,7 +132,7 @@ "# Then, query it at each point of the rectangular grid.\n", "from sklearn.neighbors import RadiusNeighborsClassifier\n", "data_estimator = RadiusNeighborsClassifier(radius=data_interpolation_radius, outlier_label=np.nan)\n", - "data_estimator.fit(sensor[['latitude','longitude']], sensor[sensor_channel])\n", + "data_estimator.fit(sensor[['latitude','longitude']], sensor[sensor_channel].astype(np.float))\n", "data_zv = data_estimator.predict(data_xy)\n", "data_zv = data_zv.reshape(data_shape).T\n", "\n", From 15d830c4e8267eafd5c405a63e51145a7c616f43 Mon Sep 17 00:00:00 2001 From: Pras Velagapudi Date: Thu, 1 Dec 2016 20:15:24 -0500 Subject: [PATCH 16/43] Only plot vehicle position for sensor readings. --- notebooks/Data_Interpolation.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/Data_Interpolation.ipynb b/notebooks/Data_Interpolation.ipynb index 89ef43c..a5a6d1b 100644 --- a/notebooks/Data_Interpolation.ipynb +++ b/notebooks/Data_Interpolation.ipynb @@ -100,7 +100,7 @@ "outputs": [], "source": [ "# Create a trail of the vehicle's path on the map.\n", - "pl = Polyline(locations=position.as_matrix().tolist())\n", + "pl = Polyline(locations=sensor[['latitude','longitude']].as_matrix().tolist())\n", "pl.fill_opacity = 0.0\n", "pl.weight = 2" ] From b6d2d1667f3e7b85e97fdef5c75dd6a8dc972382 Mon Sep 17 00:00:00 2001 From: jjblum Date: Wed, 3 May 2017 16:16:34 +0200 Subject: [PATCH 17/43] Modified data interpolation notebook to ignore data from times where EC equals exactly zero. --- examples/EC_zero_trim.py | 45 +++++++++ notebooks/Data_Interpolation.ipynb | 142 ++++++++++++++++++++++++----- 2 files changed, 162 insertions(+), 25 deletions(-) create mode 100644 examples/EC_zero_trim.py diff --git a/examples/EC_zero_trim.py b/examples/EC_zero_trim.py new file mode 100644 index 0000000..f4a17c9 --- /dev/null +++ b/examples/EC_zero_trim.py @@ -0,0 +1,45 @@ +import matplotlib.pyplot as plt +import platypus.io.logs +import platypus.util.conversions +import numpy as np + +PATH = "/home/jason/Documents/INTCATCH/phone logs/lorenzo dataset/" +FILE = "platypus_20170420_050906.txt" + + +def main(): + global PATH, FILE + print "\nLoading all the data in " + PATH + FILE + "\n" + data = platypus.io.logs.load(PATH + FILE) + if "ES2" in data: + print "ES2 sensor is present. Trimming all data within EC = 0 time windows\n" + # find all time windows where EC is exactly 0 + ES2_data = data["ES2"] + values = ES2_data["ec"].values + ec_eq_zero_indices = np.where(values == 0)[0] + windows = list() + windows.append([ec_eq_zero_indices[0]]) + left = ec_eq_zero_indices[0] + for ii in range(1, ec_eq_zero_indices.shape[0]): + i = ec_eq_zero_indices[ii] + if i - left > 5: + # there has been a jump in index, a new time window has started + windows[-1].append(left) + windows.append([i]) + left = i + windows[-1].append(ec_eq_zero_indices[-1]) + # print ec_eq_zero_indices + # print windows + for window in windows: + time_window = [ES2_data["ec"].index.values[window[0]], ES2_data["ec"].index.values[window[1]]] + for k in data.keys(): + data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])] + + else: + print "No ES2 sensor present. No trimming will be performed." + + # do stuff with data + + +if __name__ == "__main__": + main() diff --git a/notebooks/Data_Interpolation.ipynb b/notebooks/Data_Interpolation.ipynb index a5a6d1b..755a166 100644 --- a/notebooks/Data_Interpolation.ipynb +++ b/notebooks/Data_Interpolation.ipynb @@ -2,9 +2,11 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 51, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -27,16 +29,58 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 52, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ES2 sensor is present. Trimming all data within EC = 0 time windows\n", + "\n", + "Available sensors/channels:\n", + " ATLAS_DO, do\n", + " ES2, ec\n", + " ES2, temperature\n" + ] + } + ], "source": [ "# Import the data from the specified logfile\n", - "log_filename = '../logs/platypus_20161126_161922.txt'\n", + "log_filename = '/home/jason/Documents/INTCATCH/phone logs/lorenzo dataset/platypus_20170420_072302.txt'\n", + "\n", "data = platypus.io.logs.load(log_filename)\n", "\n", + "if \"ES2\" in data:\n", + " print \"ES2 sensor is present. Trimming all data within EC = 0 time windows\\n\"\n", + " # find all time windows where EC is exactly 0\n", + " ES2_data = data[\"ES2\"]\n", + " values = ES2_data[\"ec\"].values\n", + " ec_eq_zero_indices = np.where(values == 0)[0]\n", + " windows = list()\n", + " windows.append([ec_eq_zero_indices[0]])\n", + " left = ec_eq_zero_indices[0]\n", + " for ii in range(1, ec_eq_zero_indices.shape[0]):\n", + " i = ec_eq_zero_indices[ii]\n", + " if i - left > 5:\n", + " # there has been a jump in index, a new time window has started\n", + " windows[-1].append(left)\n", + " windows.append([i])\n", + " left = i\n", + " windows[-1].append(ec_eq_zero_indices[-1])\n", + " # print ec_eq_zero_indices\n", + " # print windows\n", + " for window in windows:\n", + " time_window = [ES2_data[\"ec\"].index.values[window[0]], ES2_data[\"ec\"].index.values[window[1]]]\n", + " for k in data.keys():\n", + " data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])]\n", + "else:\n", + " print \"No ES2 sensor present. No trimming will be performed.\"\n", + "\n", "# Define useful access variables.\n", "pose = data['pose']\n", "position = pose[['latitude', 'longitude']]\n", @@ -52,23 +96,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 53, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ "# Select the sensor and the name of the channel for that sensor.\n", - "sensor_name = 'HDS_DEPTH'\n", - "sensor_channel = 1\n", - "sensor_units = 'meters (m)'" + "sensor_name = 'ES2'\n", + "sensor_channel = 'temperature'\n", + "#sensor_units = 'Electrical Conductivity (uS/cm)'\n", + "sensor_units = 'Temperature (C)'\n", + "#sensor_units = 'Dissolved Oxygen (mg/L)'" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 54, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true, + "scrolled": true }, "outputs": [], "source": [ @@ -93,9 +144,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 55, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -107,9 +160,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 56, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -130,8 +185,11 @@ "\n", "# Create a radial-basis interpolator over the sensor dataset\n", "# Then, query it at each point of the rectangular grid.\n", - "from sklearn.neighbors import RadiusNeighborsClassifier\n", - "data_estimator = RadiusNeighborsClassifier(radius=data_interpolation_radius, outlier_label=np.nan)\n", + "#from sklearn.neighbors import RadiusNeighborsClassifier\n", + "#data_estimator = RadiusNeighborsClassifier(radius=data_interpolation_radius, outlier_label=np.nan)\n", + "from sklearn.neighbors import RadiusNeighborsRegressor\n", + "data_estimator = RadiusNeighborsRegressor(radius=data_interpolation_radius)\n", + "\n", "data_estimator.fit(sensor[['latitude','longitude']], sensor[sensor_channel].astype(np.float))\n", "data_zv = data_estimator.predict(data_xy)\n", "data_zv = data_zv.reshape(data_shape).T\n", @@ -160,16 +218,39 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 57, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, - "outputs": [], + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+EAAABRCAYAAAC9m6cOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAC3hJREFUeJzt3XusZWdZx/Hvb2aonWIjl0nFUmqhUtpYzVjaQoAam2Iz\nokkBIy1IQsQGlUvUGIjxloHwhyHcjMYbtQr+QYuJBQOUwRLaBoK0QzPttICWNBOZsWUsCLFS287M\n4x97jd2zzr6ts/dZZ885308y6Xr3e1nv2n3Oe/Zz1tprpaqQJEmSJElrb8t6T0CSJEmSpM3CJFyS\nJEmSpJ6YhEuSJEmS1BOTcEmSJEmSemISLkmSJElST0zCJUmSJEnqiUm4JEmSJEk9MQmXJEmSJKkn\nJuGSJEmSJPVkW5fGyY8VfP94qV07odyl7aLHatts81qgRb4984x9sow1z9iOtbY/Mr2NVVP6nlif\nCe3bdZPaDoo1cnuUdv2k9ivr2n2ntV/dfru2n2fsZZlH17EXOdZmeL+6jb36OG/Xr+sx1dA8Jv8o\nr6xvG//2rCxP6jut/bS2a9W3Xb/IY/T96rbvtRyrS9+ubRfYd0V1jd4eWZ4wjXZ910Oa53/bWr7V\nazmvaeH0IOypql0jqk7QKQkfJOC/NqbrUyYMPamua/20vosca72OcZFjdTT8yWJrx922y8P95+nb\n7j9P33b7efq2yyfr+7WWY00qL/K9n3de87z3K8pDS/K2o626E8tbth45sfopJ9ZvHWq/dVu7rtW3\nXb9lqC+tuhXl1lgT2s/Td23HWn3fdv/ufRc51vjjmGce3ceaJyaWJb7mjYnli6/O8zjaKh8ZmsfR\nY606JpbTWs5OmMqRCXUAk/q2y13aTtuX81q/ec3TdzPMq2PfJ1r1Qz/KPHF0fN3Ivq1dPTFme1rb\nafXz9G3Xt+sWOdYijxFgN+wY8fIKXo4uSZIkSVJPTMIlSZIkSeqJSbgkSZIkST0xCZckSZIkqScm\n4ZIkSZIk9cQkXJIkSZKknpiES5IkSZLUE5NwSZIkSZJ6YhIuSZIkSVJPTMIlSZIkSeqJSbgkSZIk\nST0xCZckSZIkqScm4ZIkSZIk9cQkXJIkSZKknpiES5IkSZLUE5NwSZIkSZJ6YhIuSZIkSVJPTMIl\nSZIkSeqJSbgkSZIkST0xCZckSZIkqScm4ZIkSZIk9cQkXJIkSZKknpiES5IkSZLUE5NwSZIkSZJ6\nYhIuSZIkSVJPTMIlSZIkSeqJSbgkSZIkST0xCZckSZIkqScm4ZIkSZIk9cQkXJIkSZKknpiES5Ik\nSZLUk1TV7I2TzwA7Ooy/A3i466SkJWIMayMwjrURGMfaCIxjbQTG8XgPV9WuaY06JeFdJdlbVRev\n2Q6kNWYMayMwjrURGMfaCIxjbQTG8fy8HF2SJEmSpJ6YhEuSJEmS1JO1TsL/eo3Hl9aaMayNwDjW\nRmAcayMwjrURGMdzWtPvhEuSJEmSpCd5ObokSZIkST1ZVRKe5Pokh5PcO6Lud5JUkpGPMktyNMm+\n5t8/rWb/0rxGxXCS3UkODcXnK8b03ZXkX5N8I8nv9jdr6URzxvGBJPubNnv7m7V0onGfKZK8LcnX\nk9yX5D1j+roeaynMGceux1oKYz5X3Dj0meJAkn1j+roed7Cqy9GT/DTwCPCRqrpw6PXnANcB5wMv\nrKoVz49L8khV/eDqpyzNb1QMJ9kNPFJV753Qbyvwb8DPAgeBO4HXVtVX13zSUstq47hpdwC4eNQ6\nLfVpTBxfDvw+8PNV9ViSM6rqcKuf67GWxmrjuGl3ANdjLYFxOd5Q/fuA71XVu1qvux53tKoz4VV1\nO/CdEVUfAN4B+EVzLbUJMTzNpcA3quqBqnocuAG4aqGTk2Y0RxxLS2NMHP8G8MdV9VjTZkXiguux\nlsgccSwtjUmfK5IEeA3w0RHVrscdLew74UmuAg5V1d1Tmp6aZG+Sf0nyykXtX1qQtya5p7kc5+kj\n6p8NfHOofLB5TVom0+IYBn8s/WySryR5U5+Tk2ZwHnBZki8nuS3JJSPauB5r2c0Sx+B6rJPDZcC3\nqur+EXWuxx0tJAlPchrwe8AfzdD8R6vqYuB1wAeTnLuIOUgL8BfAucBO4EHgfes7HWlVZo3jl1XV\nRcDPAW9pLkGTlsU24BnAi4G3Ax9rzsJIJ5NZ49j1WCeD1zL6LLhWYVFnws8Fngvc3Xyv5SzgriTP\najesqkPNfx8AbgV+akFzkOZSVd+qqqNVdQz4EINLa9oOAc8ZKp/VvCYthRnjeHgtPgzcNK6dtE4O\nAv9YA3cAx4D2DV9dj7XsZolj12MtvSTbgFcDN45p4nrc0UKS8KraX1VnVNU5VXUOg0Xnoqp6aLhd\nkqcn+YFmewfwUsAv7GspJPmRoeKrgBV3/2dwo4nnJ3luklOAawDv8q+lMUscJ3lqktOPbwNXjmon\nraOPA5cDJDkPOAVo37TK9VjLbmocux7rJPFy4OtVdXBMvetxR6t9RNlHgS8BL0hyMMmvTmh7cZLr\nmuIFwN4kdwOfZ3CzCpNw9W5MDL+neUTIPQx+af520/bMJJ8GqKojwFuBPcDXgI9V1X3rchDa9FYb\nx8APA19o1uI7gE9V1WfW4RCkcXF8PfC85jE5NwBvqKpyPdayWm0c43qsJTIhx7uG1qXorsfzWdUj\nyiRJkiRJUncLuzu6JEmSJEmazCRckiRJkqSemIRLkiRJktQTk3BJkiRJknpiEi5JkiRJUk9MwiVJ\naknyzCT7mn8PJTk0VD5lvec3SpI3JnnWGo7/1CS3JtnSlM9PcnOS+5PcleSGJGck2Znkb9ZqHpIk\nney2rfcEJElaNlX1bWAnQJLdwCNV9d51ndRgLlur6uiY6jcCdwEPdRhvW/N811lcC/xDVR1Lchrw\nKeBtVfXpZqwrgGdW1b4kz0vy7Ko6NOtcJEnaLDwTLklSB0nekOSO5qz4nyfZkmRbku8meX+S+5Ls\nSfKiJLcleSDJK5q+1ya5qXn9/iR/MOO4H0xyD3BpkncmuTPJvUn+MgNXM/ijwY3Hz9YnOZjkac3Y\nL05yS7P97iQfSfJF4O+afby/2fc9Sa4dc+i/DHyi2X49cNvxBBygqj5XVV9rip8Erl7cuy5J0sZh\nEi5J0oySXAi8CnhJVe1kcEXZNU31DwE3V9WPA48Du4ErgF8C3jU0zKXAKxkkza9rLt+eNu7tVfWT\nVfUl4E+q6hLgJ5q6XVV1I7APuLqqdlbV41MO5Xzgiqp6PfAm4HBVXQpcArwlydmt4z4VOKuqDjYv\nXQh8ZcL4e4HLpsxBkqRNycvRJUma3csZJKp7kwBsB77Z1D1aVf/cbO8HvldVR5LsB84ZGmNPVf0X\nQJKPAy9j8Pt43LiPAzcN9b8iyduBU4EdDJLhmzsexyeq6n+b7SuBC5IMJ/3PB/59qP0ZwHc6jH8Y\nOLPjnCRJ2hRMwiVJml2A66vqD094MdnGIFk+7hjw2ND28O/bao1ZU8Z9tKqqKZ8G/BlwUVUdSvJu\nBsn4KEd48oq3dpv/aR3Tm6vqc2PGAXi0NcZ9wIsmtD+16SNJklq8HF2SpNndArwmyQ74/7uonz2l\nT9uVSZ7WJNRXAV/sMO52Bkn9w0lOB35xqO6/gdOHygeAFzbbw+3a9gBvbhJ+krwgyfbhBlX1n8D2\noTvD/z3wM0l2HW+T5PIkFzTF84B7J+xTkqRNyzPhkiTNqKr2J3kncEvzqK4ngF8H/qPDMHcyuMHZ\nmcCHq2ofwCzjVtW3k3wY+CrwIPDloeq/Ba5L8iiD753vBj6U5LvA7RPm81fA2cC+5lL4wwz+ONB2\nC/AS4Naq+n6SXwA+kORPm/nuA36zaXs5T97ETZIkDUlzhZskSVpjzZ3HL6yq31rvuXSV5BIGl63/\nypR224HPAy+d8Dg1SZI2LS9HlyRJU1XVncAXmjP1k5wNvMMEXJKk0TwTLkmSJElSTzwTLkmSJElS\nT0zCJUmSJEnqiUm4JEmSJEk9MQmXJEmSJKknJuGSJEmSJPXEJFySJEmSpJ78H7QzloXGrmI2AAAA\nAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "beeb56b3516948019002b743a554cdd8" + } + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# Create a map centered on this data log.\n", "center = [pose['latitude'].median(), pose['longitude'].median()]\n", - "zoom = 18\n", - "m = Map(center=center, zoom=zoom, height='600px')\n", + "# print center\n", + "zoom = 17\n", + "m = Map(center=center, zoom=zoom, height='800px')\n", "m += io # Add image overlay\n", "m += pl # Add vehicle trail\n", "\n", @@ -189,6 +270,17 @@ "pyplot.show()\n", "m" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [] } ], "metadata": { @@ -207,7 +299,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", - "version": "2.7.9" + "version": "2.7.12" }, "widgets": { "state": { From 12a502c00707e5902897030d8525ef6791b901c2 Mon Sep 17 00:00:00 2001 From: jjblum Date: Tue, 3 Oct 2017 15:56:30 +0200 Subject: [PATCH 18/43] Added .csv file output of the sensor DataFrame. --- notebooks/Data_Interpolation.ipynb | 252 ++++++++++++++++++++--------- 1 file changed, 174 insertions(+), 78 deletions(-) diff --git a/notebooks/Data_Interpolation.ipynb b/notebooks/Data_Interpolation.ipynb index 755a166..0289005 100644 --- a/notebooks/Data_Interpolation.ipynb +++ b/notebooks/Data_Interpolation.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 51, + "execution_count": 27, "metadata": { "collapsed": false, "deletable": true, @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 28, "metadata": { "collapsed": false, "deletable": true, @@ -43,7 +43,6 @@ "ES2 sensor is present. Trimming all data within EC = 0 time windows\n", "\n", "Available sensors/channels:\n", - " ATLAS_DO, do\n", " ES2, ec\n", " ES2, temperature\n" ] @@ -51,16 +50,19 @@ ], "source": [ "# Import the data from the specified logfile\n", - "log_filename = '/home/jason/Documents/INTCATCH/phone logs/lorenzo dataset/platypus_20170420_072302.txt'\n", + "log_path = \"/home/jason/Documents/INTCATCH/phone logs/Gardaland outlet/2017-9-20/\"\n", + "log_filename = \"platypus_20170920_081505\"\n", + "log_ext = \".txt\"\n", "\n", - "data = platypus.io.logs.load(log_filename)\n", + "data = platypus.io.logs.load(log_path + log_filename + log_ext)\n", "\n", "if \"ES2\" in data:\n", " print \"ES2 sensor is present. Trimming all data within EC = 0 time windows\\n\"\n", " # find all time windows where EC is exactly 0\n", " ES2_data = data[\"ES2\"]\n", " values = ES2_data[\"ec\"].values\n", - " ec_eq_zero_indices = np.where(values == 0)[0]\n", + " #ec_eq_zero_indices = np.where(values == 0)[0]\n", + " ec_eq_zero_indices = np.where(values < 100)[0]\n", " windows = list()\n", " windows.append([ec_eq_zero_indices[0]])\n", " left = ec_eq_zero_indices[0]\n", @@ -76,10 +78,33 @@ " # print windows\n", " for window in windows:\n", " time_window = [ES2_data[\"ec\"].index.values[window[0]], ES2_data[\"ec\"].index.values[window[1]]]\n", - " for k in data.keys():\n", + " for k in data:\n", " data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])]\n", "else:\n", " print \"No ES2 sensor present. No trimming will be performed.\"\n", + " \n", + "\"\"\"\n", + "if \"ATLAS_PH\" in data:\n", + " print \"pH sensor is present. Trimming all data within pH < 6 time windows\\n\"\n", + " # find all time windows where pH is less than 6\n", + " pH_data = data[\"ATLAS_PH\"]\n", + " values = pH_data[\"ph\"].values\n", + " pH_lt_6_indices = np.where(values < 6)[0]\n", + " windows = list()\n", + " windows.append([pH_lt_6_indices[0]])\n", + " left = pH_lt_6_indices[0]\n", + " for ii in range(1, pH_lt_6_indices.shape[0]):\n", + " i = pH_lt_6_indices[ii]\n", + " if i - left > 5:\n", + " windows[-1].append(left)\n", + " windows.append([i])\n", + " left = i\n", + " windows[-1].append(pH_lt_6_indices[-1])\n", + " for window in windows:\n", + " time_window = [pH_data[\"ph\"].index.values[window[0]], pH_data[\"ph\"].index.values[window[1]]]\n", + " for k in data:\n", + " data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])]\n", + "\"\"\"\n", "\n", "# Define useful access variables.\n", "pose = data['pose']\n", @@ -96,7 +121,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 29, "metadata": { "collapsed": true, "deletable": true, @@ -106,45 +131,122 @@ "source": [ "# Select the sensor and the name of the channel for that sensor.\n", "sensor_name = 'ES2'\n", - "sensor_channel = 'temperature'\n", - "#sensor_units = 'Electrical Conductivity (uS/cm)'\n", - "sensor_units = 'Temperature (C)'\n", - "#sensor_units = 'Dissolved Oxygen (mg/L)'" + "sensor_channel = 'ec'\n", + "sensor_units = 'Electrical Conductivity (uS/cm)'\n", + "#sensor_units = 'Temperature (C)'\n", + "#sensor_units = 'Dissolved Oxygen (mg/L)'\n", + "#sensor_units = \"pH\"" ] }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 30, "metadata": { "collapsed": false, "deletable": true, "editable": true, "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " ec temperature latitude longitude\n", + "time \n", + "2017-09-20 12:23:25.319 1453 23.9 45.458897 10.704719\n", + "2017-09-20 12:23:28.320 1449 23.9 45.458898 10.704719\n", + "2017-09-20 12:23:31.321 1461 23.8 45.458897 10.704715\n", + "2017-09-20 12:23:34.322 1444 23.7 45.458899 10.704712\n", + "2017-09-20 12:23:37.323 1446 23.7 45.458899 10.704711\n", + "2017-09-20 12:23:40.324 1442 23.6 45.458897 10.704710\n", + "2017-09-20 12:23:43.325 1457 23.5 45.458897 10.704710\n", + "2017-09-20 12:23:46.326 1458 23.4 45.458897 10.704710\n", + "2017-09-20 12:23:49.327 1468 23.3 45.458897 10.704710\n", + "2017-09-20 12:23:52.328 1474 23.3 45.458897 10.704710\n", + "2017-09-20 12:23:55.329 1477 23.3 45.458897 10.704710\n", + "2017-09-20 12:23:58.331 1463 23.3 45.458898 10.704705\n", + "2017-09-20 12:24:01.331 1462 23.3 45.458901 10.704701\n", + "2017-09-20 12:24:04.335 1471 23.1 45.458903 10.704702\n", + "2017-09-20 12:24:07.333 1497 23.1 45.458904 10.704702\n", + "2017-09-20 12:24:10.337 1479 23.1 45.458904 10.704701\n", + "2017-09-20 12:24:13.335 1473 23.1 45.458904 10.704701\n", + "2017-09-20 12:24:16.336 1481 23.1 45.458904 10.704701\n", + "2017-09-20 12:24:19.338 1485 23.1 45.458904 10.704701\n", + "2017-09-20 12:24:22.338 1476 23.0 45.458904 10.704701\n", + "2017-09-20 12:24:25.339 1487 23.0 45.458904 10.704701\n", + "2017-09-20 12:24:28.340 1498 22.9 45.458905 10.704701\n", + "2017-09-20 12:24:31.341 1487 22.9 45.458907 10.704699\n", + "2017-09-20 12:24:34.342 1492 22.8 45.458917 10.704688\n", + "2017-09-20 12:24:49.347 1459 22.8 45.458937 10.704687\n", + "2017-09-20 12:24:52.348 1451 22.8 45.458942 10.704674\n", + "2017-09-20 12:24:55.351 1459 22.8 45.458942 10.704661\n", + "2017-09-20 12:24:58.350 1439 22.7 45.458941 10.704651\n", + "2017-09-20 12:25:01.353 1440 22.8 45.458938 10.704646\n", + "2017-09-20 12:25:04.354 1451 22.7 45.458935 10.704640\n", + "... ... ... ... ...\n", + "2017-09-20 12:59:29.044 1404 21.7 45.458860 10.704779\n", + "2017-09-20 12:59:32.045 1423 21.7 45.458860 10.704779\n", + "2017-09-20 12:59:35.048 1425 21.6 45.458860 10.704779\n", + "2017-09-20 12:59:38.047 1403 21.7 45.458860 10.704779\n", + "2017-09-20 12:59:41.050 1403 21.7 45.458860 10.704779\n", + "2017-09-20 12:59:44.054 1405 21.7 45.458860 10.704779\n", + "2017-09-20 12:59:47.050 1433 21.7 45.458860 10.704779\n", + "2017-09-20 12:59:50.051 1417 21.7 45.458860 10.704779\n", + "2017-09-20 12:59:53.052 1415 21.7 45.458860 10.704779\n", + "2017-09-20 12:59:56.053 1405 21.7 45.458861 10.704779\n", + "2017-09-20 12:59:59.054 1411 21.7 45.458861 10.704780\n", + "2017-09-20 13:00:02.055 1428 21.7 45.458861 10.704780\n", + "2017-09-20 13:00:05.056 1420 21.7 45.458861 10.704780\n", + "2017-09-20 13:00:08.057 1423 21.7 45.458861 10.704780\n", + "2017-09-20 13:00:11.058 1438 21.7 45.458861 10.704780\n", + "2017-09-20 13:00:14.059 1420 21.7 45.458861 10.704780\n", + "2017-09-20 13:00:17.060 1412 21.7 45.458861 10.704780\n", + "2017-09-20 13:00:20.061 1417 21.7 45.458861 10.704780\n", + "2017-09-20 13:00:23.062 1420 21.7 45.458861 10.704780\n", + "2017-09-20 13:00:26.064 1414 21.7 45.458861 10.704780\n", + "2017-09-20 13:00:29.064 1408 21.7 45.458861 10.704780\n", + "2017-09-20 13:00:32.065 1442 21.7 45.458861 10.704780\n", + "2017-09-20 13:00:35.067 1426 21.7 45.458861 10.704780\n", + "2017-09-20 13:00:38.067 1434 21.7 45.458861 10.704780\n", + "2017-09-20 13:00:41.068 1422 21.7 45.458861 10.704780\n", + "2017-09-20 13:00:44.076 1404 21.7 45.458861 10.704780\n", + "2017-09-20 13:00:47.070 1435 21.7 45.458861 10.704780\n", + "2017-09-20 13:00:50.073 1410 21.7 45.458861 10.704780\n", + "2017-09-20 13:00:53.075 1414 21.7 45.458861 10.704780\n", + "2017-09-20 13:00:56.079 1413 21.7 45.458861 10.704780\n", + "\n", + "[694 rows x 4 columns]\n" + ] + } + ], "source": [ "# Extract the pose timing and the sensor data of interest.\n", "pose_times = pose.index.values.astype(np.float64)\n", "\n", - "sensor = data[sensor_name]\n", - "sensor_times = sensor.index.values.astype(np.float64)\n", + "if sensor_name in data:\n", + " sensor = data[sensor_name]\n", + " sensor_times = sensor.index.values.astype(np.float64)\n", "\n", - "# Linearly interpolate the position of the sensor at every sample.\n", - "sensor_pose_interpolator = scipy.interpolate.interp1d(pose_times, pose[['latitude', 'longitude']],\n", - " axis=0, bounds_error=False)\n", + " # Linearly interpolate the position of the sensor at every sample.\n", + " sensor_pose_interpolator = scipy.interpolate.interp1d(pose_times, position,\n", + " axis=0, bounds_error=False)\n", "\n", - "# Add the position information back to the sensor data.\n", - "sensor = sensor.join(pandas.DataFrame(sensor_pose_interpolator(sensor_times), sensor.index,\n", - " columns=('latitude', 'longitude')))\n", + " # Add the position information back to the sensor data.\n", + " sensor = sensor.join(pandas.DataFrame(sensor_pose_interpolator(sensor_times), sensor.index,\n", + " columns=('latitude', 'longitude')))\n", + " \n", + " print sensor\n", + " sensor.to_csv(log_path + log_filename + \"__\" + sensor_name + \".csv\")\n", "\n", - "# Remove columns that have NaN values (no pose information).\n", - "sensor_valid = np.all(np.isfinite(sensor), axis=1)\n", - "sensor = sensor[sensor_valid]" + " # Remove columns that have NaN values (no pose information).\n", + " sensor_valid = np.all(np.isfinite(sensor), axis=1)\n", + " sensor = sensor[sensor_valid]" ] }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 31, "metadata": { "collapsed": false, "deletable": true, @@ -153,14 +255,14 @@ "outputs": [], "source": [ "# Create a trail of the vehicle's path on the map.\n", - "pl = Polyline(locations=sensor[['latitude','longitude']].as_matrix().tolist())\n", + "pl = Polyline(locations=position.as_matrix().tolist())\n", "pl.fill_opacity = 0.0\n", "pl.weight = 2" ] }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 32, "metadata": { "collapsed": false, "deletable": true, @@ -183,42 +285,43 @@ "data_shape = data_xv.shape\n", "data_xy = np.vstack([data_xv.ravel(), data_yv.ravel()]).T\n", "\n", - "# Create a radial-basis interpolator over the sensor dataset\n", - "# Then, query it at each point of the rectangular grid.\n", - "#from sklearn.neighbors import RadiusNeighborsClassifier\n", - "#data_estimator = RadiusNeighborsClassifier(radius=data_interpolation_radius, outlier_label=np.nan)\n", - "from sklearn.neighbors import RadiusNeighborsRegressor\n", - "data_estimator = RadiusNeighborsRegressor(radius=data_interpolation_radius)\n", + "if sensor_name in data:\n", + " # Create a radial-basis interpolator over the sensor dataset\n", + " # Then, query it at each point of the rectangular grid.\n", + " #from sklearn.neighbors import RadiusNeighborsClassifier\n", + " #data_estimator = RadiusNeighborsClassifier(radius=data_interpolation_radius, outlier_label=np.nan)\n", + " from sklearn.neighbors import RadiusNeighborsRegressor\n", + " data_estimator = RadiusNeighborsRegressor(radius=data_interpolation_radius)\n", "\n", - "data_estimator.fit(sensor[['latitude','longitude']], sensor[sensor_channel].astype(np.float))\n", - "data_zv = data_estimator.predict(data_xy)\n", - "data_zv = data_zv.reshape(data_shape).T\n", + " data_estimator.fit(sensor[['latitude','longitude']], sensor[sensor_channel].astype(np.float))\n", + " data_zv = data_estimator.predict(data_xy)\n", + " data_zv = data_zv.reshape(data_shape).T\n", "\n", - "# Normalize data from [0, 1)\n", - "data_max = data_zv[np.isfinite(data_zv)].max()\n", - "data_min = data_zv[np.isfinite(data_zv)].min()\n", - "data_zv = (data_zv - data_min) / (data_max - data_min)\n", + " # Normalize data from [0, 1)\n", + " data_max = data_zv[np.isfinite(data_zv)].max()\n", + " data_min = data_zv[np.isfinite(data_zv)].min()\n", + " data_zv = (data_zv - data_min) / (data_max - data_min)\n", "\n", - "# Update a color map only at the points that have valid values.\n", - "data_rgb = np.zeros((data_shape[0], data_shape[1], 4), dtype=np.uint8)\n", - "data_rgb = matplotlib.cm.jet(data_zv)*255.0\n", - "data_rgb[:,:,3] = 255 * np.isfinite(data_zv)\n", + " # Update a color map only at the points that have valid values.\n", + " data_rgb = np.zeros((data_shape[0], data_shape[1], 4), dtype=np.uint8)\n", + " data_rgb = matplotlib.cm.jet(data_zv)*255.0\n", + " data_rgb[:,:,3] = 255 * np.isfinite(data_zv)\n", "\n", - "# Remove any old image files.\n", - "old_png_files = glob.glob('./*.png')\n", - "for old_png_file in old_png_files:\n", - " os.remove(old_png_file)\n", + " # Remove any old image files.\n", + " old_png_files = glob.glob('./*.png')\n", + " for old_png_file in old_png_files:\n", + " os.remove(old_png_file)\n", "\n", - "png_filename = './platypus_data_{:s}.png'.format(uuid.uuid4())\n", - "scipy.misc.imsave(png_filename, data_rgb)\n", + " png_filename = './platypus_data_{:s}.png'.format(uuid.uuid4())\n", + " scipy.misc.imsave(png_filename, data_rgb)\n", "\n", - "# Create image overlay that references generated image.\n", - "io = ImageOverlay(url=png_filename, bounds=data_bounds)" + " # Create image overlay that references generated image.\n", + " io = ImageOverlay(url=png_filename, bounds=data_bounds)" ] }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 33, "metadata": { "collapsed": false, "deletable": true, @@ -227,9 +330,9 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+EAAABRCAYAAAC9m6cOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAC3hJREFUeJzt3XusZWdZx/Hvb2aonWIjl0nFUmqhUtpYzVjaQoAam2Iz\nokkBIy1IQsQGlUvUGIjxloHwhyHcjMYbtQr+QYuJBQOUwRLaBoK0QzPttICWNBOZsWUsCLFS287M\n4x97jd2zzr6ts/dZZ885308y6Xr3e1nv2n3Oe/Zz1tprpaqQJEmSJElrb8t6T0CSJEmSpM3CJFyS\nJEmSpJ6YhEuSJEmS1BOTcEmSJEmSemISLkmSJElST0zCJUmSJEnqiUm4JEmSJEk9MQmXJEmSJKkn\nJuGSJEmSJPVkW5fGyY8VfP94qV07odyl7aLHatts81qgRb4984x9sow1z9iOtbY/Mr2NVVP6nlif\nCe3bdZPaDoo1cnuUdv2k9ivr2n2ntV/dfru2n2fsZZlH17EXOdZmeL+6jb36OG/Xr+sx1dA8Jv8o\nr6xvG//2rCxP6jut/bS2a9W3Xb/IY/T96rbvtRyrS9+ubRfYd0V1jd4eWZ4wjXZ910Oa53/bWr7V\nazmvaeH0IOypql0jqk7QKQkfJOC/NqbrUyYMPamua/20vosca72OcZFjdTT8yWJrx922y8P95+nb\n7j9P33b7efq2yyfr+7WWY00qL/K9n3de87z3K8pDS/K2o626E8tbth45sfopJ9ZvHWq/dVu7rtW3\nXb9lqC+tuhXl1lgT2s/Td23HWn3fdv/ufRc51vjjmGce3ceaJyaWJb7mjYnli6/O8zjaKh8ZmsfR\nY606JpbTWs5OmMqRCXUAk/q2y13aTtuX81q/ec3TdzPMq2PfJ1r1Qz/KPHF0fN3Ivq1dPTFme1rb\nafXz9G3Xt+sWOdYijxFgN+wY8fIKXo4uSZIkSVJPTMIlSZIkSeqJSbgkSZIkST0xCZckSZIkqScm\n4ZIkSZIk9cQkXJIkSZKknpiES5IkSZLUE5NwSZIkSZJ6YhIuSZIkSVJPTMIlSZIkSeqJSbgkSZIk\nST0xCZckSZIkqScm4ZIkSZIk9cQkXJIkSZKknpiES5IkSZLUE5NwSZIkSZJ6YhIuSZIkSVJPTMIl\nSZIkSeqJSbgkSZIkST0xCZckSZIkqScm4ZIkSZIk9cQkXJIkSZKknpiES5IkSZLUE5NwSZIkSZJ6\nYhIuSZIkSVJPTMIlSZIkSeqJSbgkSZIkST0xCZckSZIkqScm4ZIkSZIk9cQkXJIkSZKknpiES5Ik\nSZLUk1TV7I2TzwA7Ooy/A3i466SkJWIMayMwjrURGMfaCIxjbQTG8XgPV9WuaY06JeFdJdlbVRev\n2Q6kNWYMayMwjrURGMfaCIxjbQTG8fy8HF2SJEmSpJ6YhEuSJEmS1JO1TsL/eo3Hl9aaMayNwDjW\nRmAcayMwjrURGMdzWtPvhEuSJEmSpCd5ObokSZIkST1ZVRKe5Pokh5PcO6Lud5JUkpGPMktyNMm+\n5t8/rWb/0rxGxXCS3UkODcXnK8b03ZXkX5N8I8nv9jdr6URzxvGBJPubNnv7m7V0onGfKZK8LcnX\nk9yX5D1j+roeaynMGceux1oKYz5X3Dj0meJAkn1j+roed7Cqy9GT/DTwCPCRqrpw6PXnANcB5wMv\nrKoVz49L8khV/eDqpyzNb1QMJ9kNPFJV753Qbyvwb8DPAgeBO4HXVtVX13zSUstq47hpdwC4eNQ6\nLfVpTBxfDvw+8PNV9ViSM6rqcKuf67GWxmrjuGl3ANdjLYFxOd5Q/fuA71XVu1qvux53tKoz4VV1\nO/CdEVUfAN4B+EVzLbUJMTzNpcA3quqBqnocuAG4aqGTk2Y0RxxLS2NMHP8G8MdV9VjTZkXiguux\nlsgccSwtjUmfK5IEeA3w0RHVrscdLew74UmuAg5V1d1Tmp6aZG+Sf0nyykXtX1qQtya5p7kc5+kj\n6p8NfHOofLB5TVom0+IYBn8s/WySryR5U5+Tk2ZwHnBZki8nuS3JJSPauB5r2c0Sx+B6rJPDZcC3\nqur+EXWuxx0tJAlPchrwe8AfzdD8R6vqYuB1wAeTnLuIOUgL8BfAucBO4EHgfes7HWlVZo3jl1XV\nRcDPAW9pLkGTlsU24BnAi4G3Ax9rzsJIJ5NZ49j1WCeD1zL6LLhWYVFnws8Fngvc3Xyv5SzgriTP\najesqkPNfx8AbgV+akFzkOZSVd+qqqNVdQz4EINLa9oOAc8ZKp/VvCYthRnjeHgtPgzcNK6dtE4O\nAv9YA3cAx4D2DV9dj7XsZolj12MtvSTbgFcDN45p4nrc0UKS8KraX1VnVNU5VXUOg0Xnoqp6aLhd\nkqcn+YFmewfwUsAv7GspJPmRoeKrgBV3/2dwo4nnJ3luklOAawDv8q+lMUscJ3lqktOPbwNXjmon\nraOPA5cDJDkPOAVo37TK9VjLbmocux7rJPFy4OtVdXBMvetxR6t9RNlHgS8BL0hyMMmvTmh7cZLr\nmuIFwN4kdwOfZ3CzCpNw9W5MDL+neUTIPQx+af520/bMJJ8GqKojwFuBPcDXgI9V1X3rchDa9FYb\nx8APA19o1uI7gE9V1WfW4RCkcXF8PfC85jE5NwBvqKpyPdayWm0c43qsJTIhx7uG1qXorsfzWdUj\nyiRJkiRJUncLuzu6JEmSJEmazCRckiRJkqSemIRLkiRJktQTk3BJkiRJknpiEi5JkiRJUk9MwiVJ\naknyzCT7mn8PJTk0VD5lvec3SpI3JnnWGo7/1CS3JtnSlM9PcnOS+5PcleSGJGck2Znkb9ZqHpIk\nney2rfcEJElaNlX1bWAnQJLdwCNV9d51ndRgLlur6uiY6jcCdwEPdRhvW/N811lcC/xDVR1Lchrw\nKeBtVfXpZqwrgGdW1b4kz0vy7Ko6NOtcJEnaLDwTLklSB0nekOSO5qz4nyfZkmRbku8meX+S+5Ls\nSfKiJLcleSDJK5q+1ya5qXn9/iR/MOO4H0xyD3BpkncmuTPJvUn+MgNXM/ijwY3Hz9YnOZjkac3Y\nL05yS7P97iQfSfJF4O+afby/2fc9Sa4dc+i/DHyi2X49cNvxBBygqj5XVV9rip8Erl7cuy5J0sZh\nEi5J0oySXAi8CnhJVe1kcEXZNU31DwE3V9WPA48Du4ErgF8C3jU0zKXAKxkkza9rLt+eNu7tVfWT\nVfUl4E+q6hLgJ5q6XVV1I7APuLqqdlbV41MO5Xzgiqp6PfAm4HBVXQpcArwlydmt4z4VOKuqDjYv\nXQh8ZcL4e4HLpsxBkqRNycvRJUma3csZJKp7kwBsB77Z1D1aVf/cbO8HvldVR5LsB84ZGmNPVf0X\nQJKPAy9j8Pt43LiPAzcN9b8iyduBU4EdDJLhmzsexyeq6n+b7SuBC5IMJ/3PB/59qP0ZwHc6jH8Y\nOLPjnCRJ2hRMwiVJml2A66vqD094MdnGIFk+7hjw2ND28O/bao1ZU8Z9tKqqKZ8G/BlwUVUdSvJu\nBsn4KEd48oq3dpv/aR3Tm6vqc2PGAXi0NcZ9wIsmtD+16SNJklq8HF2SpNndArwmyQ74/7uonz2l\nT9uVSZ7WJNRXAV/sMO52Bkn9w0lOB35xqO6/gdOHygeAFzbbw+3a9gBvbhJ+krwgyfbhBlX1n8D2\noTvD/z3wM0l2HW+T5PIkFzTF84B7J+xTkqRNyzPhkiTNqKr2J3kncEvzqK4ngF8H/qPDMHcyuMHZ\nmcCHq2ofwCzjVtW3k3wY+CrwIPDloeq/Ba5L8iiD753vBj6U5LvA7RPm81fA2cC+5lL4wwz+ONB2\nC/AS4Naq+n6SXwA+kORPm/nuA36zaXs5T97ETZIkDUlzhZskSVpjzZ3HL6yq31rvuXSV5BIGl63/\nypR224HPAy+d8Dg1SZI2LS9HlyRJU1XVncAXmjP1k5wNvMMEXJKk0TwTLkmSJElSTzwTLkmSJElS\nT0zCJUmSJEnqiUm4JEmSJEk9MQmXJEmSJKknJuGSJEmSJPXEJFySJEmSpJ78H7QzloXGrmI2AAAA\nAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+EAAABRCAYAAAC9m6cOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADxRJREFUeJzt3XuQJlV5x/Hvj10EQbm5URFMIIFooSBBMFhYXpAgKhGr\n4gWiEdQyWjGKJlYCatSgprCSCmJiqPKCKDEIUrHcYASJoFaIC3JbFhR14w3IChgQL1iwuzz5o8/A\nO+++c3l3Znp2Mt9P1db06XPp8/acPbtP9+l+U1VIkiRJkqSFt91id0CSJEmSpOXCIFySJEmSpJ4Y\nhEuSJEmS1BODcEmSJEmSemIQLkmSJElSTwzCJUmSJEnqiUG4JEmSJEk9MQiXJEmSJKknBuGSJEmS\nJPVk5TiFk/0K7p1IDedOkx6n7Hy3NWy59WsezefpmUvbS6WtubRtWwv7V6a3tmqGupPzM0354bzp\nynbJGrk9ynD+dOW3zBuuO1P5rTvuuOXn0va20o9x257PtpbD+Rqv7a0f58P5i/qZaqAf0/9V3jJ/\n2NSnZ8v0dHVnKj9T2YWqO5w/n5/R8zXesReyrXHqjlt2HutukV2jt0emp+nGcP64H2kuv7aFPNUL\n2a+ZhtMGuKSqjhmRNclYQXgXgL9+iqrbT9P0dHnj5s9Udz7bWqzPOJ9tjWnwfxYrxjzscHqw/lzq\nDtefS93h8nOpO5xequdrIduaLj2f536u/ZrLud8iPTAlr9w8lDc5vd2KTZOzt5+cv2Kg/IqVw3lD\ndYfztxuoy1DeFumhtqYpP5e6C9vW1tcdrj9+3flsa+rPMZd+jN/WXMbEtjK+5jomtr3xNXY/Ng+l\nNw30Y/MDQ3lMm87QdDapK5umyQOYru5wepyyMx3Lfi1ev+ZSdzn0a8y6G4fyB/4qs3Hz1Hkj6w4d\nauMU2zOVnSl/LnWH84fz5rOt+fyMAO+BVSN2b8Hl6JIkSZIk9cQgXJIkSZKknhiES5IkSZLUE4Nw\nSZIkSZJ6YhAuSZIkSVJPDMIlSZIkSeqJQbgkSZIkST0xCJckSZIkqScG4ZIkSZIk9cQgXJIkSZKk\nnhiES5IkSZLUE4NwSZIkSZJ6YhAuSZIkSVJPDMIlSZIkSeqJQbgkSZIkST0xCJckSZIkqScG4ZIk\nSZIk9cQgXJIkSZKknhiES5IkSZLUE4NwSZIkSZJ6YhAuSZIkSVJPDMIlSZIkSeqJQbgkSZIkST0x\nCJckSZIkqScG4ZIkSZIk9cQgXJIkSZKknhiES5IkSZLUE4NwSZIkSZJ6YhAuSZIkSVJPDMIlSZIk\nSeqJQbgkSZIkST1JVc2+cHIxsGrhuqNtyCrgJ4vdCS0rjjn1yfGmvjnm1DfHnPrmmIOfVNUxMxUa\nKwjX8pHk6qo6dLH7oeXDMac+Od7UN8ec+uaYU98cc7PncnRJkiRJknpiEC5JkiRJUk8MwjWVjyx2\nB7TsOObUJ8eb+uaYU98cc+qbY26WfCZckiRJkqSeeCdckiRJkqSeGIQvY0lWJLkuyUUtvW+SK5Os\nT3J+koe1/Tu09PqWv89i9ltLU5LdklyY5OYk30ry9CR7JLk0yXfbz91b2ST5UBtzNyQ5ZLH7r6Un\nyVuT3JTkxiTnJdnReU7zKcnZSe5IcuPAvrHntSQntvLfTXLiYnwWbfumGG9/2/5dvSHJ55LsNpB3\nahtv307yvIH9x7R965Oc0vfn0NIxaswN5P15kkqyqqWd48ZgEL68nQx8ayD9AeCMqtoPuBt4bdv/\nWuDutv+MVk4a15nAxVX1ROApdGPvFODLVbU/8OWWBng+sH/788fAWf13V0tZkr2ANwOHVtWTgRXA\n8TjPaX6dAwx/H+xY81qSPYB3A78LPA1490TgLg05hy3H26XAk6vqIOA7wKkASQ6gm/Oe1Or8U7v5\nsgL4MN14PAA4oZWVRjmHLcccSR4PHA38aGC3c9wYDMKXqSR7Ay8EPtbSAY4ELmxFPgm8uG0f19K0\n/Oe28tKsJNkVeCbwcYCqur+qfsrksTU85j5VnTXAbkn27LnbWvpWAg9PshLYCdiA85zmUVV9Dbhr\naPe489rzgEur6q6qupsuqNriP73SqPFWVV+qqk0tuQbYu20fB3ymqu6rqu8D6+kCoKcB66vqe1V1\nP/CZVlbawhRzHHQXq/8CGHy5mHPcGAzCl68P0v3leaClHwX8dGAivxXYq23vBdwC0PLvaeWl2doX\nuBP4RHsE4mNJdgYeU1UbWpkfA49p2w+OuWZwPEozqqrbgL+ju0q/gW7eugbnOS28cec15zvNl9cA\nX2zbjjctiCTHAbdV1dqhLMfcGAzCl6EkxwJ3VNU1i90XLRsrgUOAs6rqd4Bf8tASTQCq+6oGv65B\n86ItdTuO7gLQ44Cd8cq7eua8pr4keQewCfj0YvdF/38l2Ql4O/Cuxe7LUmcQvjwdAbwoyQ/oliEd\nSfe87m5t2SZ0y5lua9u3AY8HaPm7Av/bZ4e15N0K3FpVV7b0hXRB+e0Ty8zbzzta/oNjrhkcj9Js\nHAV8v6rurKqNwL/SzX3Oc1po485rzneakyQnAccCr6iHvnvY8aaF8Ft0F7fXtjhib+DaJI/FMTcW\ng/BlqKpOraq9q2ofupd2XFZVrwAuB17Sip0IfL5tr25pWv5lA5O8NKOq+jFwS5IntF3PBb7J5LE1\nPOZe1d60eThwz8DyTmk2fgQcnmSn9mz3xJhzntNCG3deuwQ4OsnubQXH0W2fNKMkx9A9Xviiqrp3\nIGs1cHz75od96V6WdRXwDWD/9k0RD6P7f+Dqvvutpamq1lXVo6tqnxZH3Aoc0v6f5xw3hpUzF9Ey\n8pfAZ5K8D7iO9hKt9vPcJOvpXs5w/CL1T0vbm4BPt3/0vwe8mu5C4AVJXgv8EHhZK/vvwAvoXiRz\nbysrzVpVXZnkQuBauiWa1wEfAb6A85zmSZLzgGcDq5LcSvcG4NMZY16rqruSvJcuOAI4rapGvQhJ\ny9wU4+1UYAfg0vYuyTVV9YaquinJBXQXHzcBb6yqza2dP6ULglYAZ1fVTb1/GC0Jo8ZcVX18iuLO\ncWOIF/olSZIkSeqHy9ElSZIkSeqJQbgkSZIkST0xCJckSZIkqScG4ZIkSZIk9cQgXJIkSZKknhiE\nS5K2GUk2J7l+4M8pbf9Xkhy6Fe0dnOQF0+QfmuRDW9nXkX1Ksn2S05N8N8m1Sb6e5Plbc4wRbf9i\nK+tNOg9JXjRxbqepc1qSo9r2W5LsNOYxk+SyJLvMUO7wJFe23/e3krxnIG/7JNeOc9wpjvEf7ftp\nJUladH5PuCRpW/Krqjp4Hts7GDiU7vtLJ0mysqquBq6ex+MBvBfYE3hyVd2X5DHAs+b5GOOadB6q\najWweroKVfWugeRbgH+m++7X2XoBsLaqfjZDuU8CL6uqtUlWAE8YyHsGcMUYx5zKucCfAO+fh7Yk\nSZoT74RLkpaUJEe3u8vXJvlskke0/Ycl+a8ka5NclWRX4DTg5e0u68uTvCfJuUmuAM5N8uwkF7X6\nj0jyiSTrktyQ5A/a/rOSXJ3kpiR/PUPfdgJeB7ypqu4DqKrbq+qCln9Ca//GJB8YqPeLJO9vfV/T\nAneS7Ns+67ok7xso/2C/W/ofk5w0xnk4qdXZNckPk2zX6u6c5JZ2B/qcJC9J8mbgccDlSS5P8pok\nHxw49uuSnDHidLwC+Hwrs0+SGwfqvG3gjvejgQ3tXG2uqm8OtHEM8MVW51Xt97I2yblt3znt97Mm\nyffaeTm73VE/Z6Cd1cAJ0/3uJEnqi0G4JGlb8vBMXo7+8sHMJKuAdwJHVdUhdHex/yzJw4DzgZOr\n6inAUcAvgXcB51fVwVV1fmvmgFZ/OCj7K+Ceqjqwqg4CLmv731FVhwIHAc9KctA0/d8P+NGou79J\nHgd8ADiS7s70YUle3LJ3Bta0vn+NLpAHOBM4q6oOpAWq0xnzPFBV9wDX89Cd+mOBS6pq40CZDwH/\nAzynqp4DXAD8fpLtW5FXA2eP6M4RwDUz9Rk4A/h2ks8leX2SHQfyngN8JcmT6H7vR7bPdfJAmd2B\npwNvpQu2zwCeBByY5OD2Ge4GdkjyqFn0R5KkBWUQLknalvyqBYoHDweMzeF0QfQVSa4HTgR+g24J\n84aq+gZAVf2sqjZNcYzVVfWrEfuPAj48kWiBG8DL2nPJ19EFdwds5Wc7DPhKVd3Z+vZp4Jkt735g\n4s72NcA+bfsI4Ly2fe4sjjHOeZhwPjBxseP4lp5SVf2C7gLFsUmeCGxfVetGFN2jqn4+U4er6jS6\npfJfAv4QuBggyV7AXVV1L92Fi89W1U9anbsGmvi3qipgHXB7Va2rqgeAm3joPALcQXdHX5KkReUz\n4ZKkpSTApcN3sZMcOEYbv5z1wZJ9gbcBh1XV3W2J847TVFkP/HqSXWbxLPSgjS2QBNjM5H+fa0T5\nTUy+kD5dn2ayGvibJHsAT+WhFQDT+RjwduBm4BNTlNmUZLsWEE/b36r6b+CsJB8F7mx3rI8BLplF\nX+5rPx8Y2J5ID57HHYFRF18kSeqVd8IlSUvJGuCIJPvBg88w/zbwbWDPJIe1/Y9MshL4OfDIWbZ9\nKfDGiUR7m/YudEH7Pe057Wnfct7u2n4cOLMtDSfJryV5KXAV3XL2Ve0FZCcAX52hT1fQ3Z2G7hnr\nCT8EDkiyQ5LdgOe2/WOfh3Zn+xt0S98vqqrNI4pNql9VVwKPp7tzfd6I8hN9+c22fTvw6CSPSrID\n3bJ3Wh9fmCQtuT/dRYifMvA8ON2FgZdOLCdvFwxmrbX/WOAH49STJGkhGIRLkrYlw8+Enz6YWVV3\nAicB5yW5Afg68MSqup9uSfU/JFlLF1DvCFxOF6xu8Xz5CO8Ddm8vTVtL9wz0Wrpl6DcD/8Ls3tT9\nTuBO4JvtZWQXAT+rqg3AKa1Pa4FrqurzM7R1MvDGJOuAvQbOwy10z2bf2H5e1/Zv7Xk4H3glUy9F\n/whwcZLLB/ZdAFwxsGx/2BeAZ7d+baR7OdxVrU83D5T7I7pnwq+nW3I/cbFhv6q6udW/ie7N5l9t\nn+vvpzjmVJ5K98z9TEvzJUlacHlo9ZskSdLstLezn1FVX54if0/gU1X1e1vR9jOAV1bVG+bYzYn2\nzqR7F8DIvkqS1CfvhEuSpFlLsluS79C9RG/KoLbd+f9okl3GPUZV/ed8BeDNjQbgkqRthXfCJUmS\nJEnqiXfCJUmSJEnqiUG4JEmSJEk9MQiXJEmSJKknBuGSJEmSJPXEIFySJEmSpJ4YhEuSJEmS1JP/\nA8ge5QLX9a0wAAAAAElFTkSuQmCC\n", "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -238,7 +341,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "beeb56b3516948019002b743a554cdd8" + "model_id": "e42a98992d3440bf8a0f637f5c2353e8" } }, "metadata": {}, @@ -246,41 +349,34 @@ } ], "source": [ + "\n", "# Create a map centered on this data log.\n", "center = [pose['latitude'].median(), pose['longitude'].median()]\n", "# print center\n", "zoom = 17\n", "m = Map(center=center, zoom=zoom, height='800px')\n", - "m += io # Add image overlay\n", - "m += pl # Add vehicle trail\n", + "if sensor_name in data:\n", + " m += io # Add image overlay\n", + "if sensor_name not in data:\n", + " m += pl # Add vehicle trail, but only if there isn't heatmap data to look at\n", "\n", "# Make a figure and axes with dimensions as desired.\n", "fig = pyplot.figure(figsize=(15, 3))\n", "ax1 = fig.add_axes([0.05, 0.80, 0.9, 0.15])\n", "\n", - "# Set the colormap and norm to correspond to the data for which\n", - "# the colorbar will be used.\n", - "cmap = matplotlib.cm.jet\n", - "norm = matplotlib.colors.Normalize(vmin=data_min, vmax=data_max)\n", - "cb1 = matplotlib.colorbar.ColorbarBase(ax1, cmap=cmap,\n", - " norm=norm,\n", - " orientation='horizontal')\n", - "cb1.set_label(sensor_units)\n", + "if sensor_name in data:\n", + " # Set the colormap and norm to correspond to the data for which\n", + " # the colorbar will be used.\n", + " cmap = matplotlib.cm.jet\n", + " norm = matplotlib.colors.Normalize(vmin=data_min, vmax=data_max)\n", + " cb1 = matplotlib.colorbar.ColorbarBase(ax1, cmap=cmap,\n", + " norm=norm,\n", + " orientation='horizontal')\n", + " cb1.set_label(sensor_units)\n", "\n", "pyplot.show()\n", "m" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, - "outputs": [], - "source": [] } ], "metadata": { From b4d82461c3260caf1610b56ae6b4eadc97a63ce7 Mon Sep 17 00:00:00 2001 From: jjblum Date: Tue, 3 Oct 2017 19:19:14 +0200 Subject: [PATCH 19/43] Started a prototype for extracting data collected while a sampler jar is being filled. --- examples/EC_zero_trim.py | 126 +++++++++++++++++++++++++++-- notebooks/Data_Interpolation.ipynb | 94 +++------------------ 2 files changed, 130 insertions(+), 90 deletions(-) diff --git a/examples/EC_zero_trim.py b/examples/EC_zero_trim.py index f4a17c9..dd04e0a 100644 --- a/examples/EC_zero_trim.py +++ b/examples/EC_zero_trim.py @@ -1,22 +1,29 @@ -import matplotlib.pyplot as plt +import collections import platypus.io.logs import platypus.util.conversions import numpy as np +import datetime +import json +import six +import re +import pandas -PATH = "/home/jason/Documents/INTCATCH/phone logs/lorenzo dataset/" -FILE = "platypus_20170420_050906.txt" +PATH = "/home/jason/Documents/INTCATCH/phone logs/Gardaland outlet/2017-10-3/" +FILE = "platypus_20171003_050016" +EXT = ".txt" -def main(): +""" +def trim_EC(): global PATH, FILE print "\nLoading all the data in " + PATH + FILE + "\n" data = platypus.io.logs.load(PATH + FILE) if "ES2" in data: - print "ES2 sensor is present. Trimming all data within EC = 0 time windows\n" + print "ES2 sensor is present. Trimming all data within EC < 100 time windows\n" # find all time windows where EC is exactly 0 ES2_data = data["ES2"] values = ES2_data["ec"].values - ec_eq_zero_indices = np.where(values == 0)[0] + ec_eq_zero_indices = np.where(values < 100)[0] windows = list() windows.append([ec_eq_zero_indices[0]]) left = ec_eq_zero_indices[0] @@ -39,7 +46,112 @@ def main(): print "No ES2 sensor present. No trimming will be performed." # do stuff with data +""" + + +def trim_using_EC(dataframe, threshold=100): + """ + Trims any data when EC < 100 + :return: trimmed dataframe + """ + if "ES2" in dataframe: + print "ES2 sensor is present. Trimming all data within EC < {:.0f} time windows\n".format(threshold) + # find all time windows where EC is exactly 0 + ES2_data = dataframe["ES2"] + values = ES2_data["ec"].values + ec_eq_zero_indices = np.where(values < threshold)[0] + windows = list() + windows.append([ec_eq_zero_indices[0]]) + left = ec_eq_zero_indices[0] + for ii in range(1, ec_eq_zero_indices.shape[0]): + i = ec_eq_zero_indices[ii] + if i - left > 5: + # there has been a jump in index, a new time window has started + windows[-1].append(left) + windows.append([i]) + left = i + windows[-1].append(ec_eq_zero_indices[-1]) + # print ec_eq_zero_indices + # print windows + for window in windows: + time_window = [ES2_data["ec"].index.values[window[0]], ES2_data["ec"].index.values[window[1]]] + for k in dataframe: + dataframe[k] = dataframe[k].loc[np.logical_or(dataframe[k].index < time_window[0], dataframe[k].index > time_window[1])] + else: + print "No ES2 sensor present. No trimming will be performed." + return dataframe + + +def data_with_sampler(): + global PATH, FILE + filename = PATH + FILE + EXT + data = platypus.io.logs.load(filename) + is_EC_gt_100 = False + + jar_start_timestamps = dict() + with open(filename, 'r') as logfile: + raw_data = collections.defaultdict(list) + start_time = datetime.datetime.utcfromtimestamp(0) + + for line in logfile: + # Extract each line fron the logfile and convert the timestamp. + time_offset_ms, level, message = line.split('\t', 2) + + # Compute the timestamp for each log entry. + time_offset = datetime.timedelta(milliseconds=int(time_offset_ms)) + timestamp = start_time + time_offset + + # Try to parse the log as a JSON object. + try: + entry = json.loads(message) + except ValueError as e: + raise ValueError( + "Aborted after invalid JSON log message '{:s}': {:s}".format(message, e)) + + # If the line is a datetime, compute subsequent timestamps from this. + # We assume that "date" and "time" are always together in the entry. + if 'date' in entry: + timestamp = datetime.datetime.utcfromtimestamp(entry['time'] / 1000.) + start_time = timestamp - time_offset + + # Extract appropriate data from each entry. + for k, v in six.viewitems(entry): + if k == 'sensor': + if v['type'] == "ES2": + ec = v['data'][0] + if not is_EC_gt_100 and ec > 100: + is_EC_gt_100 = True + if is_EC_gt_100 and ec < 100: + is_EC_gt_100 = False + if k == 'sampler' and is_EC_gt_100: + if "start" in v: + # the in-water sampler start messages + m = re.search('[0-9]+', v) + jar_id = m.group(0) + jar_start_timestamps[jar_id] = timestamp + + # TODO: MUST MERGE IN THE LATITUDE AND LONGITUDE!!! + + return data, jar_start_timestamps if __name__ == "__main__": - main() + global PATH, FILE + filename = PATH + FILE + EXT + + data, jar_start_timestamps = data_with_sampler() + trimmed_data = trim_using_EC(data) + for k in jar_start_timestamps: + start_time = jar_start_timestamps[k] + end_time = start_time + datetime.timedelta(minutes=3.75) + print "Jar {} lasts from {} to {}".format(k, start_time, end_time) + for sensor in data.keys(): + print sensor + if sensor not in ["ES2", "ATLAS_DO", "ATLAS_PH"]: + continue + dataframe = trimmed_data[sensor] + relevantframe = dataframe.between_time(start_time.time(), end_time.time()) + output_filename = PATH + FILE + "__JAR_{}".format(k) + "__{}".format(sensor) + ".csv" + relevantframe.to_csv(output_filename) + + diff --git a/notebooks/Data_Interpolation.ipynb b/notebooks/Data_Interpolation.ipynb index 0289005..f5bd1d6 100644 --- a/notebooks/Data_Interpolation.ipynb +++ b/notebooks/Data_Interpolation.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 27, + "execution_count": 34, "metadata": { "collapsed": false, "deletable": true, @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 35, "metadata": { "collapsed": false, "deletable": true, @@ -121,7 +121,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 36, "metadata": { "collapsed": true, "deletable": true, @@ -140,86 +140,14 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 37, "metadata": { "collapsed": false, "deletable": true, "editable": true, "scrolled": true }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " ec temperature latitude longitude\n", - "time \n", - "2017-09-20 12:23:25.319 1453 23.9 45.458897 10.704719\n", - "2017-09-20 12:23:28.320 1449 23.9 45.458898 10.704719\n", - "2017-09-20 12:23:31.321 1461 23.8 45.458897 10.704715\n", - "2017-09-20 12:23:34.322 1444 23.7 45.458899 10.704712\n", - "2017-09-20 12:23:37.323 1446 23.7 45.458899 10.704711\n", - "2017-09-20 12:23:40.324 1442 23.6 45.458897 10.704710\n", - "2017-09-20 12:23:43.325 1457 23.5 45.458897 10.704710\n", - "2017-09-20 12:23:46.326 1458 23.4 45.458897 10.704710\n", - "2017-09-20 12:23:49.327 1468 23.3 45.458897 10.704710\n", - "2017-09-20 12:23:52.328 1474 23.3 45.458897 10.704710\n", - "2017-09-20 12:23:55.329 1477 23.3 45.458897 10.704710\n", - "2017-09-20 12:23:58.331 1463 23.3 45.458898 10.704705\n", - "2017-09-20 12:24:01.331 1462 23.3 45.458901 10.704701\n", - "2017-09-20 12:24:04.335 1471 23.1 45.458903 10.704702\n", - "2017-09-20 12:24:07.333 1497 23.1 45.458904 10.704702\n", - "2017-09-20 12:24:10.337 1479 23.1 45.458904 10.704701\n", - "2017-09-20 12:24:13.335 1473 23.1 45.458904 10.704701\n", - "2017-09-20 12:24:16.336 1481 23.1 45.458904 10.704701\n", - "2017-09-20 12:24:19.338 1485 23.1 45.458904 10.704701\n", - "2017-09-20 12:24:22.338 1476 23.0 45.458904 10.704701\n", - "2017-09-20 12:24:25.339 1487 23.0 45.458904 10.704701\n", - "2017-09-20 12:24:28.340 1498 22.9 45.458905 10.704701\n", - "2017-09-20 12:24:31.341 1487 22.9 45.458907 10.704699\n", - "2017-09-20 12:24:34.342 1492 22.8 45.458917 10.704688\n", - "2017-09-20 12:24:49.347 1459 22.8 45.458937 10.704687\n", - "2017-09-20 12:24:52.348 1451 22.8 45.458942 10.704674\n", - "2017-09-20 12:24:55.351 1459 22.8 45.458942 10.704661\n", - "2017-09-20 12:24:58.350 1439 22.7 45.458941 10.704651\n", - "2017-09-20 12:25:01.353 1440 22.8 45.458938 10.704646\n", - "2017-09-20 12:25:04.354 1451 22.7 45.458935 10.704640\n", - "... ... ... ... ...\n", - "2017-09-20 12:59:29.044 1404 21.7 45.458860 10.704779\n", - "2017-09-20 12:59:32.045 1423 21.7 45.458860 10.704779\n", - "2017-09-20 12:59:35.048 1425 21.6 45.458860 10.704779\n", - "2017-09-20 12:59:38.047 1403 21.7 45.458860 10.704779\n", - "2017-09-20 12:59:41.050 1403 21.7 45.458860 10.704779\n", - "2017-09-20 12:59:44.054 1405 21.7 45.458860 10.704779\n", - "2017-09-20 12:59:47.050 1433 21.7 45.458860 10.704779\n", - "2017-09-20 12:59:50.051 1417 21.7 45.458860 10.704779\n", - "2017-09-20 12:59:53.052 1415 21.7 45.458860 10.704779\n", - "2017-09-20 12:59:56.053 1405 21.7 45.458861 10.704779\n", - "2017-09-20 12:59:59.054 1411 21.7 45.458861 10.704780\n", - "2017-09-20 13:00:02.055 1428 21.7 45.458861 10.704780\n", - "2017-09-20 13:00:05.056 1420 21.7 45.458861 10.704780\n", - "2017-09-20 13:00:08.057 1423 21.7 45.458861 10.704780\n", - "2017-09-20 13:00:11.058 1438 21.7 45.458861 10.704780\n", - "2017-09-20 13:00:14.059 1420 21.7 45.458861 10.704780\n", - "2017-09-20 13:00:17.060 1412 21.7 45.458861 10.704780\n", - "2017-09-20 13:00:20.061 1417 21.7 45.458861 10.704780\n", - "2017-09-20 13:00:23.062 1420 21.7 45.458861 10.704780\n", - "2017-09-20 13:00:26.064 1414 21.7 45.458861 10.704780\n", - "2017-09-20 13:00:29.064 1408 21.7 45.458861 10.704780\n", - "2017-09-20 13:00:32.065 1442 21.7 45.458861 10.704780\n", - "2017-09-20 13:00:35.067 1426 21.7 45.458861 10.704780\n", - "2017-09-20 13:00:38.067 1434 21.7 45.458861 10.704780\n", - "2017-09-20 13:00:41.068 1422 21.7 45.458861 10.704780\n", - "2017-09-20 13:00:44.076 1404 21.7 45.458861 10.704780\n", - "2017-09-20 13:00:47.070 1435 21.7 45.458861 10.704780\n", - "2017-09-20 13:00:50.073 1410 21.7 45.458861 10.704780\n", - "2017-09-20 13:00:53.075 1414 21.7 45.458861 10.704780\n", - "2017-09-20 13:00:56.079 1413 21.7 45.458861 10.704780\n", - "\n", - "[694 rows x 4 columns]\n" - ] - } - ], + "outputs": [], "source": [ "# Extract the pose timing and the sensor data of interest.\n", "pose_times = pose.index.values.astype(np.float64)\n", @@ -236,7 +164,7 @@ " sensor = sensor.join(pandas.DataFrame(sensor_pose_interpolator(sensor_times), sensor.index,\n", " columns=('latitude', 'longitude')))\n", " \n", - " print sensor\n", + " # print sensor data to csv file\n", " sensor.to_csv(log_path + log_filename + \"__\" + sensor_name + \".csv\")\n", "\n", " # Remove columns that have NaN values (no pose information).\n", @@ -246,7 +174,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 38, "metadata": { "collapsed": false, "deletable": true, @@ -262,7 +190,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 39, "metadata": { "collapsed": false, "deletable": true, @@ -321,7 +249,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 40, "metadata": { "collapsed": false, "deletable": true, @@ -332,7 +260,7 @@ "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+EAAABRCAYAAAC9m6cOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADxRJREFUeJzt3XuQJlV5x/Hvj10EQbm5URFMIIFooSBBMFhYXpAgKhGr\n4gWiEdQyWjGKJlYCatSgprCSCmJiqPKCKDEIUrHcYASJoFaIC3JbFhR14w3IChgQL1iwuzz5o8/A\nO+++c3l3Znp2Mt9P1db06XPp8/acPbtP9+l+U1VIkiRJkqSFt91id0CSJEmSpOXCIFySJEmSpJ4Y\nhEuSJEmS1BODcEmSJEmSemIQLkmSJElSTwzCJUmSJEnqiUG4JEmSJEk9MQiXJEmSJKknBuGSJEmS\nJPVk5TiFk/0K7p1IDedOkx6n7Hy3NWy59WsezefpmUvbS6WtubRtWwv7V6a3tmqGupPzM0354bzp\nynbJGrk9ynD+dOW3zBuuO1P5rTvuuOXn0va20o9x257PtpbD+Rqv7a0f58P5i/qZaqAf0/9V3jJ/\n2NSnZ8v0dHVnKj9T2YWqO5w/n5/R8zXesReyrXHqjlt2HutukV2jt0emp+nGcP64H2kuv7aFPNUL\n2a+ZhtMGuKSqjhmRNclYQXgXgL9+iqrbT9P0dHnj5s9Udz7bWqzPOJ9tjWnwfxYrxjzscHqw/lzq\nDtefS93h8nOpO5xequdrIduaLj2f536u/ZrLud8iPTAlr9w8lDc5vd2KTZOzt5+cv2Kg/IqVw3lD\ndYfztxuoy1DeFumhtqYpP5e6C9vW1tcdrj9+3flsa+rPMZd+jN/WXMbEtjK+5jomtr3xNXY/Ng+l\nNw30Y/MDQ3lMm87QdDapK5umyQOYru5wepyyMx3Lfi1ev+ZSdzn0a8y6G4fyB/4qs3Hz1Hkj6w4d\nauMU2zOVnSl/LnWH84fz5rOt+fyMAO+BVSN2b8Hl6JIkSZIk9cQgXJIkSZKknhiES5IkSZLUE4Nw\nSZIkSZJ6YhAuSZIkSVJPDMIlSZIkSeqJQbgkSZIkST0xCJckSZIkqScG4ZIkSZIk9cQgXJIkSZKk\nnhiES5IkSZLUE4NwSZIkSZJ6YhAuSZIkSVJPDMIlSZIkSeqJQbgkSZIkST0xCJckSZIkqScG4ZIk\nSZIk9cQgXJIkSZKknhiES5IkSZLUE4NwSZIkSZJ6YhAuSZIkSVJPDMIlSZIkSeqJQbgkSZIkST0x\nCJckSZIkqScG4ZIkSZIk9cQgXJIkSZKknhiES5IkSZLUE4NwSZIkSZJ6YhAuSZIkSVJPDMIlSZIk\nSeqJQbgkSZIkST1JVc2+cHIxsGrhuqNtyCrgJ4vdCS0rjjn1yfGmvjnm1DfHnPrmmIOfVNUxMxUa\nKwjX8pHk6qo6dLH7oeXDMac+Od7UN8ec+uaYU98cc7PncnRJkiRJknpiEC5JkiRJUk8MwjWVjyx2\nB7TsOObUJ8eb+uaYU98cc+qbY26WfCZckiRJkqSeeCdckiRJkqSeGIQvY0lWJLkuyUUtvW+SK5Os\nT3J+koe1/Tu09PqWv89i9ltLU5LdklyY5OYk30ry9CR7JLk0yXfbz91b2ST5UBtzNyQ5ZLH7r6Un\nyVuT3JTkxiTnJdnReU7zKcnZSe5IcuPAvrHntSQntvLfTXLiYnwWbfumGG9/2/5dvSHJ55LsNpB3\nahtv307yvIH9x7R965Oc0vfn0NIxaswN5P15kkqyqqWd48ZgEL68nQx8ayD9AeCMqtoPuBt4bdv/\nWuDutv+MVk4a15nAxVX1ROApdGPvFODLVbU/8OWWBng+sH/788fAWf13V0tZkr2ANwOHVtWTgRXA\n8TjPaX6dAwx/H+xY81qSPYB3A78LPA1490TgLg05hy3H26XAk6vqIOA7wKkASQ6gm/Oe1Or8U7v5\nsgL4MN14PAA4oZWVRjmHLcccSR4PHA38aGC3c9wYDMKXqSR7Ay8EPtbSAY4ELmxFPgm8uG0f19K0\n/Oe28tKsJNkVeCbwcYCqur+qfsrksTU85j5VnTXAbkn27LnbWvpWAg9PshLYCdiA85zmUVV9Dbhr\naPe489rzgEur6q6qupsuqNriP73SqPFWVV+qqk0tuQbYu20fB3ymqu6rqu8D6+kCoKcB66vqe1V1\nP/CZVlbawhRzHHQXq/8CGHy5mHPcGAzCl68P0v3leaClHwX8dGAivxXYq23vBdwC0PLvaeWl2doX\nuBP4RHsE4mNJdgYeU1UbWpkfA49p2w+OuWZwPEozqqrbgL+ju0q/gW7eugbnOS28cec15zvNl9cA\nX2zbjjctiCTHAbdV1dqhLMfcGAzCl6EkxwJ3VNU1i90XLRsrgUOAs6rqd4Bf8tASTQCq+6oGv65B\n86ItdTuO7gLQ44Cd8cq7eua8pr4keQewCfj0YvdF/38l2Ql4O/Cuxe7LUmcQvjwdAbwoyQ/oliEd\nSfe87m5t2SZ0y5lua9u3AY8HaPm7Av/bZ4e15N0K3FpVV7b0hXRB+e0Ty8zbzzta/oNjrhkcj9Js\nHAV8v6rurKqNwL/SzX3Oc1po485rzneakyQnAccCr6iHvnvY8aaF8Ft0F7fXtjhib+DaJI/FMTcW\ng/BlqKpOraq9q2ofupd2XFZVrwAuB17Sip0IfL5tr25pWv5lA5O8NKOq+jFwS5IntF3PBb7J5LE1\nPOZe1d60eThwz8DyTmk2fgQcnmSn9mz3xJhzntNCG3deuwQ4OsnubQXH0W2fNKMkx9A9Xviiqrp3\nIGs1cHz75od96V6WdRXwDWD/9k0RD6P7f+Dqvvutpamq1lXVo6tqnxZH3Aoc0v6f5xw3hpUzF9Ey\n8pfAZ5K8D7iO9hKt9vPcJOvpXs5w/CL1T0vbm4BPt3/0vwe8mu5C4AVJXgv8EHhZK/vvwAvoXiRz\nbysrzVpVXZnkQuBauiWa1wEfAb6A85zmSZLzgGcDq5LcSvcG4NMZY16rqruSvJcuOAI4rapGvQhJ\ny9wU4+1UYAfg0vYuyTVV9YaquinJBXQXHzcBb6yqza2dP6ULglYAZ1fVTb1/GC0Jo8ZcVX18iuLO\ncWOIF/olSZIkSeqHy9ElSZIkSeqJQbgkSZIkST0xCJckSZIkqScG4ZIkSZIk9cQgXJIkSZKknhiE\nS5K2GUk2J7l+4M8pbf9Xkhy6Fe0dnOQF0+QfmuRDW9nXkX1Ksn2S05N8N8m1Sb6e5Plbc4wRbf9i\nK+tNOg9JXjRxbqepc1qSo9r2W5LsNOYxk+SyJLvMUO7wJFe23/e3krxnIG/7JNeOc9wpjvEf7ftp\nJUladH5PuCRpW/Krqjp4Hts7GDiU7vtLJ0mysqquBq6ex+MBvBfYE3hyVd2X5DHAs+b5GOOadB6q\najWweroKVfWugeRbgH+m++7X2XoBsLaqfjZDuU8CL6uqtUlWAE8YyHsGcMUYx5zKucCfAO+fh7Yk\nSZoT74RLkpaUJEe3u8vXJvlskke0/Ycl+a8ka5NclWRX4DTg5e0u68uTvCfJuUmuAM5N8uwkF7X6\nj0jyiSTrktyQ5A/a/rOSXJ3kpiR/PUPfdgJeB7ypqu4DqKrbq+qCln9Ca//GJB8YqPeLJO9vfV/T\nAneS7Ns+67ok7xso/2C/W/ofk5w0xnk4qdXZNckPk2zX6u6c5JZ2B/qcJC9J8mbgccDlSS5P8pok\nHxw49uuSnDHidLwC+Hwrs0+SGwfqvG3gjvejgQ3tXG2uqm8OtHEM8MVW51Xt97I2yblt3znt97Mm\nyffaeTm73VE/Z6Cd1cAJ0/3uJEnqi0G4JGlb8vBMXo7+8sHMJKuAdwJHVdUhdHex/yzJw4DzgZOr\n6inAUcAvgXcB51fVwVV1fmvmgFZ/OCj7K+Ceqjqwqg4CLmv731FVhwIHAc9KctA0/d8P+NGou79J\nHgd8ADiS7s70YUle3LJ3Bta0vn+NLpAHOBM4q6oOpAWq0xnzPFBV9wDX89Cd+mOBS6pq40CZDwH/\nAzynqp4DXAD8fpLtW5FXA2eP6M4RwDUz9Rk4A/h2ks8leX2SHQfyngN8JcmT6H7vR7bPdfJAmd2B\npwNvpQu2zwCeBByY5OD2Ge4GdkjyqFn0R5KkBWUQLknalvyqBYoHDweMzeF0QfQVSa4HTgR+g24J\n84aq+gZAVf2sqjZNcYzVVfWrEfuPAj48kWiBG8DL2nPJ19EFdwds5Wc7DPhKVd3Z+vZp4Jkt735g\n4s72NcA+bfsI4Ly2fe4sjjHOeZhwPjBxseP4lp5SVf2C7gLFsUmeCGxfVetGFN2jqn4+U4er6jS6\npfJfAv4QuBggyV7AXVV1L92Fi89W1U9anbsGmvi3qipgHXB7Va2rqgeAm3joPALcQXdHX5KkReUz\n4ZKkpSTApcN3sZMcOEYbv5z1wZJ9gbcBh1XV3W2J847TVFkP/HqSXWbxLPSgjS2QBNjM5H+fa0T5\nTUy+kD5dn2ayGvibJHsAT+WhFQDT+RjwduBm4BNTlNmUZLsWEE/b36r6b+CsJB8F7mx3rI8BLplF\nX+5rPx8Y2J5ID57HHYFRF18kSeqVd8IlSUvJGuCIJPvBg88w/zbwbWDPJIe1/Y9MshL4OfDIWbZ9\nKfDGiUR7m/YudEH7Pe057Wnfct7u2n4cOLMtDSfJryV5KXAV3XL2Ve0FZCcAX52hT1fQ3Z2G7hnr\nCT8EDkiyQ5LdgOe2/WOfh3Zn+xt0S98vqqrNI4pNql9VVwKPp7tzfd6I8hN9+c22fTvw6CSPSrID\n3bJ3Wh9fmCQtuT/dRYifMvA8ON2FgZdOLCdvFwxmrbX/WOAH49STJGkhGIRLkrYlw8+Enz6YWVV3\nAicB5yW5Afg68MSqup9uSfU/JFlLF1DvCFxOF6xu8Xz5CO8Ddm8vTVtL9wz0Wrpl6DcD/8Ls3tT9\nTuBO4JvtZWQXAT+rqg3AKa1Pa4FrqurzM7R1MvDGJOuAvQbOwy10z2bf2H5e1/Zv7Xk4H3glUy9F\n/whwcZLLB/ZdAFwxsGx/2BeAZ7d+baR7OdxVrU83D5T7I7pnwq+nW3I/cbFhv6q6udW/ie7N5l9t\nn+vvpzjmVJ5K98z9TEvzJUlacHlo9ZskSdLstLezn1FVX54if0/gU1X1e1vR9jOAV1bVG+bYzYn2\nzqR7F8DIvkqS1CfvhEuSpFlLsluS79C9RG/KoLbd+f9okl3GPUZV/ed8BeDNjQbgkqRthXfCJUmS\nJEnqiXfCJUmSJEnqiUG4JEmSJEk9MQiXJEmSJKknBuGSJEmSJPXEIFySJEmSpJ4YhEuSJEmS1JP/\nA8ge5QLX9a0wAAAAAElFTkSuQmCC\n", "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -341,7 +269,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e42a98992d3440bf8a0f637f5c2353e8" + "model_id": "67ccec35ebc04778aa9bc3beec367b86" } }, "metadata": {}, From d52736f1dfa8633acbcff0328d1022264fff494a Mon Sep 17 00:00:00 2001 From: jjblum Date: Wed, 4 Oct 2017 18:22:23 +0200 Subject: [PATCH 20/43] Added merging of files. --- examples/EC_zero_trim.py | 52 ++++++++++++++++++++++++++---- notebooks/Data_Interpolation.ipynb | 40 +++++++++++++++-------- src/platypus/io/logs.py | 42 ++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 21 deletions(-) diff --git a/examples/EC_zero_trim.py b/examples/EC_zero_trim.py index dd04e0a..7185e4a 100644 --- a/examples/EC_zero_trim.py +++ b/examples/EC_zero_trim.py @@ -8,10 +8,15 @@ import re import pandas +# FILE TO TEST JAR DATA EXTRACTION PATH = "/home/jason/Documents/INTCATCH/phone logs/Gardaland outlet/2017-10-3/" FILE = "platypus_20171003_050016" EXT = ".txt" +# FILES TO TEST MERGING +PATH2 = "/home/jason/Documents/INTCATCH/phone logs/Gardaland outlet/2017-10-4/" +FILE1 = "platypus_20171004_040203" +FILE2 = "platypus_20171004_054619" """ def trim_EC(): @@ -49,6 +54,36 @@ def trim_EC(): """ +def merge_files(filename_list): + """ + + :param: filename_list: list of full path filename strings + :return: One result will all the dataframes merged + :rtype: {str: pandas.DataFrame} + """ + logfile_result_list = [platypus.io.logs.load(filename) for filename in filename_list] + if len(logfile_result_list) == 1: + return logfile_result_list[0] + all_data_types = set() + for i in range(1, len(logfile_result_list)): + all_data_types = all_data_types.union(set(logfile_result_list[i].keys())) + print all_data_types + + # merged_dataframe = pandas.DataFrame.merge(merged_dataframe[data_type], dataframe_list[i][data_type], how='outer') + merged_dataframe_dict = dict() + + for data_type in all_data_types: + for i in range(len(logfile_result_list)): + if data_type in logfile_result_list[i]: + first_log_index = i + break + merged_dataframe_dict[data_type] = logfile_result_list[first_log_index][data_type] + for i in range(first_log_index + 1, len(logfile_result_list)): + if data_type in logfile_result_list[i]: + merged_dataframe_dict[data_type] = merged_dataframe_dict[data_type].combine_first(logfile_result_list[i][data_type]).dropna(how='any') + return merged_dataframe_dict + + def trim_using_EC(dataframe, threshold=100): """ Trims any data when EC < 100 @@ -82,9 +117,7 @@ def trim_using_EC(dataframe, threshold=100): return dataframe -def data_with_sampler(): - global PATH, FILE - filename = PATH + FILE + EXT +def data_with_sampler(filename): data = platypus.io.logs.load(filename) is_EC_gt_100 = False @@ -135,11 +168,10 @@ def data_with_sampler(): return data, jar_start_timestamps -if __name__ == "__main__": - global PATH, FILE +def extract_sampler_data_by_jar(): + global PATH, FILE, EXT filename = PATH + FILE + EXT - - data, jar_start_timestamps = data_with_sampler() + data, jar_start_timestamps = data_with_sampler(filename) trimmed_data = trim_using_EC(data) for k in jar_start_timestamps: start_time = jar_start_timestamps[k] @@ -155,3 +187,9 @@ def data_with_sampler(): relevantframe.to_csv(output_filename) +if __name__ == "__main__": + merged_data = merge_files([PATH2 + FILE1 + EXT, PATH2 + FILE2 + EXT]) + + + + diff --git a/notebooks/Data_Interpolation.ipynb b/notebooks/Data_Interpolation.ipynb index f5bd1d6..9df3bbd 100644 --- a/notebooks/Data_Interpolation.ipynb +++ b/notebooks/Data_Interpolation.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 34, + "execution_count": 27, "metadata": { "collapsed": false, "deletable": true, @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 28, "metadata": { "collapsed": false, "deletable": true, @@ -40,6 +40,7 @@ "name": "stdout", "output_type": "stream", "text": [ + "set([u'BATTERY', 'pose', u'ES2'])\n", "ES2 sensor is present. Trimming all data within EC = 0 time windows\n", "\n", "Available sensors/channels:\n", @@ -50,11 +51,13 @@ ], "source": [ "# Import the data from the specified logfile\n", - "log_path = \"/home/jason/Documents/INTCATCH/phone logs/Gardaland outlet/2017-9-20/\"\n", - "log_filename = \"platypus_20170920_081505\"\n", + "log_path = \"/home/jason/Documents/INTCATCH/phone logs/Gardaland outlet/2017-10-4/\"\n", + "log_filenames = [log_path + \"platypus_20171004_054619.txt\", log_path + \"platypus_20171004_040203.txt\"]\n", + "csv_output_filename = \"2017_10_4__LidoRonchi\"\n", "log_ext = \".txt\"\n", "\n", - "data = platypus.io.logs.load(log_path + log_filename + log_ext)\n", + "#data = platypus.io.logs.load(log_path + log_filename + log_ext)\n", + "data = platypus.io.logs.merge_files(log_filenames)\n", "\n", "if \"ES2\" in data:\n", " print \"ES2 sensor is present. Trimming all data within EC = 0 time windows\\n\"\n", @@ -121,7 +124,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 29, "metadata": { "collapsed": true, "deletable": true, @@ -140,7 +143,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 30, "metadata": { "collapsed": false, "deletable": true, @@ -165,7 +168,7 @@ " columns=('latitude', 'longitude')))\n", " \n", " # print sensor data to csv file\n", - " sensor.to_csv(log_path + log_filename + \"__\" + sensor_name + \".csv\")\n", + " sensor.to_csv(log_path + csv_output_filename + \"__\" + sensor_name + \".csv\")\n", "\n", " # Remove columns that have NaN values (no pose information).\n", " sensor_valid = np.all(np.isfinite(sensor), axis=1)\n", @@ -174,7 +177,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 31, "metadata": { "collapsed": false, "deletable": true, @@ -190,7 +193,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 32, "metadata": { "collapsed": false, "deletable": true, @@ -249,7 +252,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 35, "metadata": { "collapsed": false, "deletable": true, @@ -258,9 +261,9 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+EAAABRCAYAAAC9m6cOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADxRJREFUeJzt3XuQJlV5x/Hvj10EQbm5URFMIIFooSBBMFhYXpAgKhGr\n4gWiEdQyWjGKJlYCatSgprCSCmJiqPKCKDEIUrHcYASJoFaIC3JbFhR14w3IChgQL1iwuzz5o8/A\nO+++c3l3Znp2Mt9P1db06XPp8/acPbtP9+l+U1VIkiRJkqSFt91id0CSJEmSpOXCIFySJEmSpJ4Y\nhEuSJEmS1BODcEmSJEmSemIQLkmSJElSTwzCJUmSJEnqiUG4JEmSJEk9MQiXJEmSJKknBuGSJEmS\nJPVk5TiFk/0K7p1IDedOkx6n7Hy3NWy59WsezefpmUvbS6WtubRtWwv7V6a3tmqGupPzM0354bzp\nynbJGrk9ynD+dOW3zBuuO1P5rTvuuOXn0va20o9x257PtpbD+Rqv7a0f58P5i/qZaqAf0/9V3jJ/\n2NSnZ8v0dHVnKj9T2YWqO5w/n5/R8zXesReyrXHqjlt2HutukV2jt0emp+nGcP64H2kuv7aFPNUL\n2a+ZhtMGuKSqjhmRNclYQXgXgL9+iqrbT9P0dHnj5s9Udz7bWqzPOJ9tjWnwfxYrxjzscHqw/lzq\nDtefS93h8nOpO5xequdrIduaLj2f536u/ZrLud8iPTAlr9w8lDc5vd2KTZOzt5+cv2Kg/IqVw3lD\ndYfztxuoy1DeFumhtqYpP5e6C9vW1tcdrj9+3flsa+rPMZd+jN/WXMbEtjK+5jomtr3xNXY/Ng+l\nNw30Y/MDQ3lMm87QdDapK5umyQOYru5wepyyMx3Lfi1ev+ZSdzn0a8y6G4fyB/4qs3Hz1Hkj6w4d\nauMU2zOVnSl/LnWH84fz5rOt+fyMAO+BVSN2b8Hl6JIkSZIk9cQgXJIkSZKknhiES5IkSZLUE4Nw\nSZIkSZJ6YhAuSZIkSVJPDMIlSZIkSeqJQbgkSZIkST0xCJckSZIkqScG4ZIkSZIk9cQgXJIkSZKk\nnhiES5IkSZLUE4NwSZIkSZJ6YhAuSZIkSVJPDMIlSZIkSeqJQbgkSZIkST0xCJckSZIkqScG4ZIk\nSZIk9cQgXJIkSZKknhiES5IkSZLUE4NwSZIkSZJ6YhAuSZIkSVJPDMIlSZIkSeqJQbgkSZIkST0x\nCJckSZIkqScG4ZIkSZIk9cQgXJIkSZKknhiES5IkSZLUE4NwSZIkSZJ6YhAuSZIkSVJPDMIlSZIk\nSeqJQbgkSZIkST1JVc2+cHIxsGrhuqNtyCrgJ4vdCS0rjjn1yfGmvjnm1DfHnPrmmIOfVNUxMxUa\nKwjX8pHk6qo6dLH7oeXDMac+Od7UN8ec+uaYU98cc7PncnRJkiRJknpiEC5JkiRJUk8MwjWVjyx2\nB7TsOObUJ8eb+uaYU98cc+qbY26WfCZckiRJkqSeeCdckiRJkqSeGIQvY0lWJLkuyUUtvW+SK5Os\nT3J+koe1/Tu09PqWv89i9ltLU5LdklyY5OYk30ry9CR7JLk0yXfbz91b2ST5UBtzNyQ5ZLH7r6Un\nyVuT3JTkxiTnJdnReU7zKcnZSe5IcuPAvrHntSQntvLfTXLiYnwWbfumGG9/2/5dvSHJ55LsNpB3\nahtv307yvIH9x7R965Oc0vfn0NIxaswN5P15kkqyqqWd48ZgEL68nQx8ayD9AeCMqtoPuBt4bdv/\nWuDutv+MVk4a15nAxVX1ROApdGPvFODLVbU/8OWWBng+sH/788fAWf13V0tZkr2ANwOHVtWTgRXA\n8TjPaX6dAwx/H+xY81qSPYB3A78LPA1490TgLg05hy3H26XAk6vqIOA7wKkASQ6gm/Oe1Or8U7v5\nsgL4MN14PAA4oZWVRjmHLcccSR4PHA38aGC3c9wYDMKXqSR7Ay8EPtbSAY4ELmxFPgm8uG0f19K0\n/Oe28tKsJNkVeCbwcYCqur+qfsrksTU85j5VnTXAbkn27LnbWvpWAg9PshLYCdiA85zmUVV9Dbhr\naPe489rzgEur6q6qupsuqNriP73SqPFWVV+qqk0tuQbYu20fB3ymqu6rqu8D6+kCoKcB66vqe1V1\nP/CZVlbawhRzHHQXq/8CGHy5mHPcGAzCl68P0v3leaClHwX8dGAivxXYq23vBdwC0PLvaeWl2doX\nuBP4RHsE4mNJdgYeU1UbWpkfA49p2w+OuWZwPEozqqrbgL+ju0q/gW7eugbnOS28cec15zvNl9cA\nX2zbjjctiCTHAbdV1dqhLMfcGAzCl6EkxwJ3VNU1i90XLRsrgUOAs6rqd4Bf8tASTQCq+6oGv65B\n86ItdTuO7gLQ44Cd8cq7eua8pr4keQewCfj0YvdF/38l2Ql4O/Cuxe7LUmcQvjwdAbwoyQ/oliEd\nSfe87m5t2SZ0y5lua9u3AY8HaPm7Av/bZ4e15N0K3FpVV7b0hXRB+e0Ty8zbzzta/oNjrhkcj9Js\nHAV8v6rurKqNwL/SzX3Oc1po485rzneakyQnAccCr6iHvnvY8aaF8Ft0F7fXtjhib+DaJI/FMTcW\ng/BlqKpOraq9q2ofupd2XFZVrwAuB17Sip0IfL5tr25pWv5lA5O8NKOq+jFwS5IntF3PBb7J5LE1\nPOZe1d60eThwz8DyTmk2fgQcnmSn9mz3xJhzntNCG3deuwQ4OsnubQXH0W2fNKMkx9A9Xviiqrp3\nIGs1cHz75od96V6WdRXwDWD/9k0RD6P7f+Dqvvutpamq1lXVo6tqnxZH3Aoc0v6f5xw3hpUzF9Ey\n8pfAZ5K8D7iO9hKt9vPcJOvpXs5w/CL1T0vbm4BPt3/0vwe8mu5C4AVJXgv8EHhZK/vvwAvoXiRz\nbysrzVpVXZnkQuBauiWa1wEfAb6A85zmSZLzgGcDq5LcSvcG4NMZY16rqruSvJcuOAI4rapGvQhJ\ny9wU4+1UYAfg0vYuyTVV9YaquinJBXQXHzcBb6yqza2dP6ULglYAZ1fVTb1/GC0Jo8ZcVX18iuLO\ncWOIF/olSZIkSeqHy9ElSZIkSeqJQbgkSZIkST0xCJckSZIkqScG4ZIkSZIk9cQgXJIkSZKknhiE\nS5K2GUk2J7l+4M8pbf9Xkhy6Fe0dnOQF0+QfmuRDW9nXkX1Ksn2S05N8N8m1Sb6e5Plbc4wRbf9i\nK+tNOg9JXjRxbqepc1qSo9r2W5LsNOYxk+SyJLvMUO7wJFe23/e3krxnIG/7JNeOc9wpjvEf7ftp\nJUladH5PuCRpW/Krqjp4Hts7GDiU7vtLJ0mysqquBq6ex+MBvBfYE3hyVd2X5DHAs+b5GOOadB6q\najWweroKVfWugeRbgH+m++7X2XoBsLaqfjZDuU8CL6uqtUlWAE8YyHsGcMUYx5zKucCfAO+fh7Yk\nSZoT74RLkpaUJEe3u8vXJvlskke0/Ycl+a8ka5NclWRX4DTg5e0u68uTvCfJuUmuAM5N8uwkF7X6\nj0jyiSTrktyQ5A/a/rOSXJ3kpiR/PUPfdgJeB7ypqu4DqKrbq+qCln9Ca//GJB8YqPeLJO9vfV/T\nAneS7Ns+67ok7xso/2C/W/ofk5w0xnk4qdXZNckPk2zX6u6c5JZ2B/qcJC9J8mbgccDlSS5P8pok\nHxw49uuSnDHidLwC+Hwrs0+SGwfqvG3gjvejgQ3tXG2uqm8OtHEM8MVW51Xt97I2yblt3znt97Mm\nyffaeTm73VE/Z6Cd1cAJ0/3uJEnqi0G4JGlb8vBMXo7+8sHMJKuAdwJHVdUhdHex/yzJw4DzgZOr\n6inAUcAvgXcB51fVwVV1fmvmgFZ/OCj7K+Ceqjqwqg4CLmv731FVhwIHAc9KctA0/d8P+NGou79J\nHgd8ADiS7s70YUle3LJ3Bta0vn+NLpAHOBM4q6oOpAWq0xnzPFBV9wDX89Cd+mOBS6pq40CZDwH/\nAzynqp4DXAD8fpLtW5FXA2eP6M4RwDUz9Rk4A/h2ks8leX2SHQfyngN8JcmT6H7vR7bPdfJAmd2B\npwNvpQu2zwCeBByY5OD2Ge4GdkjyqFn0R5KkBWUQLknalvyqBYoHDweMzeF0QfQVSa4HTgR+g24J\n84aq+gZAVf2sqjZNcYzVVfWrEfuPAj48kWiBG8DL2nPJ19EFdwds5Wc7DPhKVd3Z+vZp4Jkt735g\n4s72NcA+bfsI4Ly2fe4sjjHOeZhwPjBxseP4lp5SVf2C7gLFsUmeCGxfVetGFN2jqn4+U4er6jS6\npfJfAv4QuBggyV7AXVV1L92Fi89W1U9anbsGmvi3qipgHXB7Va2rqgeAm3joPALcQXdHX5KkReUz\n4ZKkpSTApcN3sZMcOEYbv5z1wZJ9gbcBh1XV3W2J847TVFkP/HqSXWbxLPSgjS2QBNjM5H+fa0T5\nTUy+kD5dn2ayGvibJHsAT+WhFQDT+RjwduBm4BNTlNmUZLsWEE/b36r6b+CsJB8F7mx3rI8BLplF\nX+5rPx8Y2J5ID57HHYFRF18kSeqVd8IlSUvJGuCIJPvBg88w/zbwbWDPJIe1/Y9MshL4OfDIWbZ9\nKfDGiUR7m/YudEH7Pe057Wnfct7u2n4cOLMtDSfJryV5KXAV3XL2Ve0FZCcAX52hT1fQ3Z2G7hnr\nCT8EDkiyQ5LdgOe2/WOfh3Zn+xt0S98vqqrNI4pNql9VVwKPp7tzfd6I8hN9+c22fTvw6CSPSrID\n3bJ3Wh9fmCQtuT/dRYifMvA8ON2FgZdOLCdvFwxmrbX/WOAH49STJGkhGIRLkrYlw8+Enz6YWVV3\nAicB5yW5Afg68MSqup9uSfU/JFlLF1DvCFxOF6xu8Xz5CO8Ddm8vTVtL9wz0Wrpl6DcD/8Ls3tT9\nTuBO4JvtZWQXAT+rqg3AKa1Pa4FrqurzM7R1MvDGJOuAvQbOwy10z2bf2H5e1/Zv7Xk4H3glUy9F\n/whwcZLLB/ZdAFwxsGx/2BeAZ7d+baR7OdxVrU83D5T7I7pnwq+nW3I/cbFhv6q6udW/ie7N5l9t\nn+vvpzjmVJ5K98z9TEvzJUlacHlo9ZskSdLstLezn1FVX54if0/gU1X1e1vR9jOAV1bVG+bYzYn2\nzqR7F8DIvkqS1CfvhEuSpFlLsluS79C9RG/KoLbd+f9okl3GPUZV/ed8BeDNjQbgkqRthXfCJUmS\nJEnqiXfCJUmSJEnqiUG4JEmSJEk9MQiXJEmSJKknBuGSJEmSJPXEIFySJEmSpJ4YhEuSJEmS1JP/\nA8ge5QLX9a0wAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+cAAABRCAYAAACwhddJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADm5JREFUeJzt3XusZWV5x/Hvj5mREZS7WgQtEKwEBUcEg8GoXDSoFE2q\nAtV6qVWbmoo2prGtNUq10aQpQm1IvCutFDQ1UtqCVNCmVEBu43CzXiqgRcCCg4jBmeHpH+s9sGfP\nOfucPWfPrL07309ycta73st69jrv7NnPXu/aO1WFJEmSJEnqz059ByBJkiRJ0o7O5FySJEmSpJ6Z\nnEuSJEmS1DOTc0mSJEmSemZyLkmSJElSz0zOJUmSJEnqmcm5JEmSJEk9MzmXJEmSJKlnJueSJEmS\nJPVs5TiNk4MLHpwrDdeOKI/TdtJjDdvR4pqgSZ6e5Yw9K2MtZ2zH2rb/ZLbbWLVI383rM6L9cN2o\ntl2x5t2ez3D9qPZb1g33Xaz91h133PbLGXta4hh37EmOtSOcr/HG3vp5Plzf62OqgThG/1Pesn7Y\nwqdny/Kovou1X6zttuo7XD/Jx+j5Gu/Y23KscfqO23aCfbeorvm35y2PCGO4ftyHtJw/27Y81dsy\nrsWm051wSVWdOE/VkoyVnHeJ+dsW6LpqxNCj6satX6zvJMfq6zFOcqwxDb7iWDHmYYfLg/2X03e4\n/3L6DrdfTt/h8qyer2051qjyJM/9cuNazrnfojzwVL1y01Dd5uWdVmzcvHrV5vUrBtqvWDlcN9R3\nuH6ngb4M1W1RHhprRPvl9N22Y2193+H+4/ed5FgLP47lxDH+WMuZE9Myv5Y7J6Zvfo0dx6ah8saB\nODY9PFTHyHKGns42C2XjiDqAUX2Hy+O0XexYxtVfXMvpuyPENWbfDUP1A/+U2bBp4bp5+w4dasMC\n24u1Xax+OX2H64frJjnWJB8jwPthn3l2L5nL2iVJkiRJ6pnJuSRJkiRJPTM5lyRJkiSpZybnkiRJ\nkiT1zORckiRJkqSemZxLkiRJktQzk3NJkiRJknpmci5JkiRJUs9MziVJkiRJ6pnJuSRJkiRJPTM5\nlyRJkiSpZybnkiRJkiT1zORckiRJkqSemZxLkiRJktQzk3NJkiRJknpmci5JkiRJUs9MziVJkiRJ\n6pnJuSRJkiRJPTM5lyRJkiSpZybnkiRJkiT1zORckiRJkqSemZxLkiRJktQzk3NJkiRJknpmci5J\nkiRJUs9MziVJkiRJ6pnJuSRJkiRJPTM5lyRJkiSpZybnkiRJkiT1zORckiRJkqSemZxLkiRJktQz\nk3NJkiRJknqWqlp64+RiYJ9tF44maB/gp30HIY3BOatZ45zVrHHOatY4ZzVrVlfVM7e281jJuWZH\nkmuq6si+45CWyjmrWeOc1axxzmrWOGc1a5Y7Z13WLkmSJElSz0zOJUmSJEnqmcn5/18f7zsAaUzO\nWc0a56xmjXNWs8Y5q1mzrDnrPeeSJEmSJPXMK+eSJEmSJPXM5HwGJVmd5Ooka5PclOQDbf+BSa5K\n8r0k5yd5TNu/cyt/r9Uf0Gf82nElWZHk+iQXtbJzVlMryQ+TrEtyQ5Jr2r69klya5Lvt955tf5Kc\n3ebst5Mc0W/02hEl2SPJl5LcmuSWJM9zzmpaJXl6e36d+7k/yTuds5pmSd7V8q8bk5zX8rKJvZ41\nOZ9NDwHHVdWzgDXAiUmOBj4CnFlVBwP3AW9u7d8M3Nf2n9naSX04HbhloOyc1bQ7tqrWDHwtynuA\nr1XV04CvtTLAS4GntZ+3Auds90glOAu4uKoOAZ5F93zrnNVUqqrvtOfXNcBzgAeBL+Oc1ZRKsh/w\nDuDI9l3mK4BTmeDrWZPzGVSdB1pxVfsp4DjgS23/54BXtu1XtDKt/vgk2U7hSgAk2R94OfDJVg7O\nWc2ewbk5PGc/356frwT2SLJvHwFqx5Rkd+AFwKcAqupXVfUznLOaDccD36+q23DOarqtBB6bZCWw\nC3AnE3w9a3I+o9ry4BuAu4FLge8DP6uqja3Jj4D92vZ+wB0ArX49sPf2jVjio8AfAw+38t44ZzXd\nCvhqkmuTvLXte1JV3dm2fwI8qW0/MmebwfksbQ8HAvcAn2m3D30yya44ZzUbTgXOa9vOWU2lqvox\n8FfA7XRJ+XrgWib4etbkfEZV1aa2DGh/4LnAIT2HJC0oyUnA3VV1bd+xSGN4flUdQbeU8u1JXjBY\nWd3XnfiVJ5oWK4EjgHOq6tnAL3h0OTDgnNV0avfnngx8cbjOOatp0j7/4BV0b4Y+GdgVOHGSxzA5\nn3FtydrlwPPolvesbFX7Az9u2z8GngLQ6ncH/nc7h6od2zHAyUl+CPwD3fKfs3DOaoq1d8ipqrvp\n7oN8LnDX3DLK9vvu1vyROdsMzmdpe/gR8KOquqqVv0SXrDtnNe1eClxXVXe1snNW0+oE4L+r6p6q\n2gD8I91r3Im9njU5n0FJnpBkj7b9WODFdB/6cjnwqtbsDcBX2vaFrUyrv6z8gnttR1X1J1W1f1Ud\nQLd07bKqei3OWU2pJLsmefzcNvAS4EY2n5vDc/b17dOEjwbWDyzLlLa5qvoJcEeSp7ddxwM345zV\n9DuNR5e0g3NW0+t24Ogku7R7x+eeZyf2eja+3p09SQ6n+3CBFXRvsFxQVWckOYjuquRewPXA66rq\noSSrgXOBZwP3AqdW1Q/6iV47uiQvAt5dVSc5ZzWt2tz8ciuuBL5QVR9KsjdwAfBU4DbgNVV1b/tP\n+mN0y9seBN5UVdf0ELp2YEnW0H3o5mOAHwBvor1OwDmrKdTe/LwdOKiq1rd9Ps9qaqX7CutTgI10\nr11/j+7e8om8njU5lyRJkiSpZy5rlyRJkiSpZybnkiRJkiT1zORckiRJkqSemZxLkiRJktQzk3NJ\nkiRJknpmci5JmhpJNiW5YeDnPW3/15McuRXjrUnyshH1RyY5eytjnTemJKuSfDjJd5Ncl+SbSV66\nNceYZ+wHtrLfZuchyclz53ZEnzOSnNC235lklzGPmSSXJdltkXZHJ7mq/b1vSfL+gbpVSa4b57gL\nHOPfkuy53HEkSdqWVvYdgCRJA35ZVWsmON4a4EjgX4Yrkqxs35E76e/J/QtgX+CZ7XtOnwS8cMLH\nGNdm56GqLgQuHNWhqt43UHwn8Hd03y28VC8D1lbV/Yu0+xzd9xivTbICePpA3fOBK8Y45kLOBf4A\n+NAExpIkaZvwyrkkaaYkeUm7Gn1dki8meVzbf1SS/0yyNsnVSXYHzgBOaVdlT0ny/iTnJrkCODfJ\ni5Jc1Po/LslnkqxL8u0kv9X2n5PkmiQ3JfnAIrHtArwF+MOqegigqu6qqgta/Wlt/BuTfGSg3wNJ\nPtRiv7Il9CQ5sD3WdUk+OND+kbhb+WNJ3jjGeXhj67N7ktuS7NT67prkjnbF+rNJXpXkHcCTgcuT\nXJ7kd5N8dODYb0ly5jyn47XAV1qbA5LcONDn3QNXyJ8I3NnO1aaqunlgjBOBf219Xt/+LmuTnNv2\nfbb9fa5M8oN2Xj7drsB/dmCcC4HTRv3tJEnqm8m5JGmaPDabL2s/ZbAyyT7Ae4ETquoIuqvef5Tk\nMcD5wOlV9SzgBOAXwPuA86tqTVWd34Y5tPUfTtb+HFhfVYdV1eHAZW3/n1XVkcDhwAuTHD4i/oOB\n2+e7WpzkycBHgOPormQfleSVrXpX4MoW+7/TJfgAZwHnVNVhtAR2lDHPA1W1HriBR6/snwRcUlUb\nBtqcDfwPcGxVHQtcAPxmklWtyZuAT88TzjHAtYvFDJwJfCfJl5O8Lcnqgbpjga8neQbd3/249rhO\nH2izJ/A84F10SfiZwDOAw5KsaY/hPmDnJHsvIR5Jknphci5Jmia/bAnkmuFEsjmaLrm+IskNwBuA\nX6dbCn1nVX0LoKrur6qNCxzjwqr65Tz7TwD+dq7QEjqA17T7nq+nS/oO3crHdhTw9aq6p8X298AL\nWt2vgLkr4dcCB7TtY4Dz2va5SzjGOOdhzvnA3Jsgp7bygqrqAbo3Lk5KcgiwqqrWzdN0r6r6+WIB\nV9UZdEvuvwr8NnAxQJL9gHur6kG6NzS+WFU/bX3uHRjin6qqgHXAXVW1rqoeBm7i0fMIcDfdCgBJ\nkqaS95xLkmZJgEuHr3onOWyMMX6x5IMlBwLvBo6qqvvaUunVI7p8D3hqkt2WcK/1oA0twQTYxOb/\nP9c87Tey+Rvso2JazIXAXybZC3gOj64YGOWTwJ8CtwKfWaDNxiQ7tUR5ZLxV9X3gnCSfAO5pV7hP\nBC5ZQiwPtd8PD2zPlQfP42pgvjdlJEmaCl45lyTNkiuBY5IcDI/cI/0bwHeAfZMc1fY/PslK4OfA\n45c49qXA2+cK7dO9d6NL5te3+8BHfup6u8r7KeCstsScJE9I8mrgarpl8fu0Dz47DfjGIjFdQXc1\nG7p7uOfcBhyaZOckewDHt/1jn4d2JfxbdEvoL6qqTfM026x/VV0FPIXuSvd587Sfi+Wgtn0X8MQk\neyfZmW75PC3GlydJKz6N7s2JnzFwvzndGwavnluW3t5IWLI2/q8BPxynnyRJ25PJuSRpmgzfc/7h\nwcqqugd4I3Bekm8D3wQOqapf0S3N/pska+kS7dXA5XRJ7Bb3r8/jg8Ce7cPa1tLdY72Wbjn7rcAX\nWNonh78XuAe4uX0I2kXA/VV1J/CeFtNa4Nqq+soiY50OvD3JOmC/gfNwB9293ze239e3/Vt7Hs4H\nXsfCS9o/Dlyc5PKBfRcAVwws/x/2z8CLWlwb6D6U7uoW060D7X6H7p7zG+iW7s+9CXFwVd3a+t9E\n90nr32iP668XOOZCnkN3T/9iS/wlSepNHl1FJ0mStDTt0+LPrKqvLVC/L/D5qnrxVoz9fOB1VfX7\nywxzbryz6D5rYN5YJUmaBl45lyRJS5ZkjyT/RffhfQsmu22lwCeS7DbuMarqPyaVmDc3mphLkqad\nV84lSZIkSeqZV84lSZIkSeqZybkkSZIkST0zOZckSZIkqWcm55IkSZIk9czkXJIkSZKknpmcS5Ik\nSZLUs/8D8gLjmV3DLV8AAAAASUVORK5CYII=\n", "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -269,7 +272,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "67ccec35ebc04778aa9bc3beec367b86" + "model_id": "5a1e07d47f5646449dfda8110c5f39b7" } }, "metadata": {}, @@ -305,6 +308,15 @@ "pyplot.show()\n", "m" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/src/platypus/io/logs.py b/src/platypus/io/logs.py index 64780da..cda25ac 100644 --- a/src/platypus/io/logs.py +++ b/src/platypus/io/logs.py @@ -95,6 +95,48 @@ """ +def merge_files(filename_list): + """ + + :param: filename_list: list of full path filename strings + :return: One result will all the dataframes merged + :rtype: {str: pandas.DataFrame} + """ + logfile_result_list = [load(filename) for filename in filename_list] + if len(logfile_result_list) == 1: + return logfile_result_list[0] + all_data_types = set() + for i in range(1, len(logfile_result_list)): + all_data_types = all_data_types.union(set(logfile_result_list[i].keys())) + print all_data_types + + merged_dataframe_dict = dict() + + for data_type in all_data_types: + for i in range(len(logfile_result_list)): + if data_type in logfile_result_list[i]: + first_log_index = i + break + merged_dataframe_dict[data_type] = logfile_result_list[first_log_index][data_type] + for i in range(first_log_index + 1, len(logfile_result_list)): + if data_type in logfile_result_list[i]: + merged_dataframe_dict[data_type] = merged_dataframe_dict[data_type].combine_first(logfile_result_list[i][data_type]).dropna(how='any') + return merged_dataframe_dict + + +def read_around_sampler(logfile, pump_duration_seconds=4*60): + """ + Reads text logs from a Platypus vehicle server logfile, particularly focusing on data following a sampler jar activation + + :param logfile: the logfile as an iterable + :param pump_duration_seconds: an integer, the number of seconds to extract data + :return: a dict containing the data from this logfile + :rtype: {int: pandas.DataFrame}, where int key is jar number (1-4) + """ + #TODO + return + + def read_v4_2_0(logfile): """ Reads text logs from a Platypus vehicle server logfile. From cef962e2d52f47d6227a07c6e127b2c56644ce08 Mon Sep 17 00:00:00 2001 From: jjblum Date: Fri, 13 Oct 2017 11:41:01 +0200 Subject: [PATCH 21/43] Minor changes. --- examples/loading_logs.py | 10 ++--- notebooks/Data_Interpolation.ipynb | 72 ++++++++++++++---------------- src/platypus/io/logs.py | 7 +-- 3 files changed, 42 insertions(+), 47 deletions(-) diff --git a/examples/loading_logs.py b/examples/loading_logs.py index 1feb320..31fdfe8 100644 --- a/examples/loading_logs.py +++ b/examples/loading_logs.py @@ -18,23 +18,23 @@ # Read the data log from file. # Note: for log versions <5.0, this filename must be 'airboat_[timestamp].txt]. -data = platypus.io.logs.load('./airboat_20130807_063622.txt') +data = platypus.io.logs.load('/home/jason/Documents/INTCATCH/phone logs/fishing pond/platypus_20170307_053438.txt') # Access the first 100 GPS locations for the vehicle. -poses = data['pose'][100:] +poses = data['pose'] # Plot the first 100 GPS locations as UTM coordinates using matplotlib. plt.plot(poses['easting'], poses['northing']) plt.show() # Retrieve temperature data from the ES2 sensor. -temp = data['es2']['temperature'] +temp = data['ES2']['temp'] # Plot ES2 electrical conductivity data using matplotlib. -plt.plot(data['es2'].index, data['es2']['ec']) +plt.plot(data['ES2'].index, data['ES2']['ec']) plt.show() # Get the standard deviation of the ES2 data. -es2_stddev = data['es2'].std() +es2_stddev = data['ES2'].std() diff --git a/notebooks/Data_Interpolation.ipynb b/notebooks/Data_Interpolation.ipynb index 9df3bbd..2e4bab7 100644 --- a/notebooks/Data_Interpolation.ipynb +++ b/notebooks/Data_Interpolation.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 27, + "execution_count": 1, "metadata": { "collapsed": false, "deletable": true, @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 2, "metadata": { "collapsed": false, "deletable": true, @@ -40,7 +40,6 @@ "name": "stdout", "output_type": "stream", "text": [ - "set([u'BATTERY', 'pose', u'ES2'])\n", "ES2 sensor is present. Trimming all data within EC = 0 time windows\n", "\n", "Available sensors/channels:\n", @@ -51,9 +50,9 @@ ], "source": [ "# Import the data from the specified logfile\n", - "log_path = \"/home/jason/Documents/INTCATCH/phone logs/Gardaland outlet/2017-10-4/\"\n", - "log_filenames = [log_path + \"platypus_20171004_054619.txt\", log_path + \"platypus_20171004_040203.txt\"]\n", - "csv_output_filename = \"2017_10_4__LidoRonchi\"\n", + "log_path = \"/home/jason/Documents/INTCATCH/phone logs/fishing pond/2017-10-12/\"\n", + "log_filenames = [log_path + \"platypus_20171012_072334.txt\"]\n", + "csv_output_filename = \"2017_10_12__Atlantide\"\n", "log_ext = \".txt\"\n", "\n", "#data = platypus.io.logs.load(log_path + log_filename + log_ext)\n", @@ -65,7 +64,7 @@ " ES2_data = data[\"ES2\"]\n", " values = ES2_data[\"ec\"].values\n", " #ec_eq_zero_indices = np.where(values == 0)[0]\n", - " ec_eq_zero_indices = np.where(values < 100)[0]\n", + " ec_eq_zero_indices = np.where(values < 200)[0]\n", " windows = list()\n", " windows.append([ec_eq_zero_indices[0]])\n", " left = ec_eq_zero_indices[0]\n", @@ -124,7 +123,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 3, "metadata": { "collapsed": true, "deletable": true, @@ -143,7 +142,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 4, "metadata": { "collapsed": false, "deletable": true, @@ -177,7 +176,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": { "collapsed": false, "deletable": true, @@ -193,13 +192,24 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "metadata": { "collapsed": false, "deletable": true, "editable": true }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python2.7/dist-packages/numpy/core/fromnumeric.py:2909: RuntimeWarning: Mean of empty slice.\n", + " out=out, **kwargs)\n", + "/usr/local/lib/python2.7/dist-packages/numpy/core/_methods.py:73: RuntimeWarning: invalid value encountered in true_divide\n", + " ret, rcount, out=ret, casting='unsafe', subok=False)\n" + ] + } + ], "source": [ "# Add a data overlay for the map\n", "data_padding = [0.0001, 0.0001] # degrees lat/lon\n", @@ -231,7 +241,9 @@ " # Normalize data from [0, 1)\n", " data_max = data_zv[np.isfinite(data_zv)].max()\n", " data_min = data_zv[np.isfinite(data_zv)].min()\n", - " data_zv = (data_zv - data_min) / (data_max - data_min)\n", + " print \"Data min = {:f} Data max = {:f}\".format(data_min, data_max)\n", + " NORMALIZER = data_max # 800\n", + " data_zv = (data_zv - data_min) / (NORMALIZER - data_min)\n", "\n", " # Update a color map only at the points that have valid values.\n", " data_rgb = np.zeros((data_shape[0], data_shape[1], 4), dtype=np.uint8)\n", @@ -252,40 +264,20 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "metadata": { "collapsed": false, "deletable": true, "editable": true }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+cAAABRCAYAAACwhddJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADm5JREFUeJzt3XusZWV5x/Hvj5mREZS7WgQtEKwEBUcEg8GoXDSoFE2q\nAtV6qVWbmoo2prGtNUq10aQpQm1IvCutFDQ1UtqCVNCmVEBu43CzXiqgRcCCg4jBmeHpH+s9sGfP\nOfucPWfPrL07309ycta73st69jrv7NnPXu/aO1WFJEmSJEnqz059ByBJkiRJ0o7O5FySJEmSpJ6Z\nnEuSJEmS1DOTc0mSJEmSemZyLkmSJElSz0zOJUmSJEnqmcm5JEmSJEk9MzmXJEmSJKlnJueSJEmS\nJPVs5TiNk4MLHpwrDdeOKI/TdtJjDdvR4pqgSZ6e5Yw9K2MtZ2zH2rb/ZLbbWLVI383rM6L9cN2o\ntl2x5t2ez3D9qPZb1g33Xaz91h133PbLGXta4hh37EmOtSOcr/HG3vp5Plzf62OqgThG/1Pesn7Y\nwqdny/Kovou1X6zttuo7XD/Jx+j5Gu/Y23KscfqO23aCfbeorvm35y2PCGO4ftyHtJw/27Y81dsy\nrsWm051wSVWdOE/VkoyVnHeJ+dsW6LpqxNCj6satX6zvJMfq6zFOcqwxDb7iWDHmYYfLg/2X03e4\n/3L6DrdfTt/h8qyer2051qjyJM/9cuNazrnfojzwVL1y01Dd5uWdVmzcvHrV5vUrBtqvWDlcN9R3\nuH6ngb4M1W1RHhprRPvl9N22Y2193+H+4/ed5FgLP47lxDH+WMuZE9Myv5Y7J6Zvfo0dx6ah8saB\nODY9PFTHyHKGns42C2XjiDqAUX2Hy+O0XexYxtVfXMvpuyPENWbfDUP1A/+U2bBp4bp5+w4dasMC\n24u1Xax+OX2H64frJjnWJB8jwPthn3l2L5nL2iVJkiRJ6pnJuSRJkiRJPTM5lyRJkiSpZybnkiRJ\nkiT1zORckiRJkqSemZxLkiRJktQzk3NJkiRJknpmci5JkiRJUs9MziVJkiRJ6pnJuSRJkiRJPTM5\nlyRJkiSpZybnkiRJkiT1zORckiRJkqSemZxLkiRJktQzk3NJkiRJknpmci5JkiRJUs9MziVJkiRJ\n6pnJuSRJkiRJPTM5lyRJkiSpZybnkiRJkiT1zORckiRJkqSemZxLkiRJktQzk3NJkiRJknpmci5J\nkiRJUs9MziVJkiRJ6pnJuSRJkiRJPTM5lyRJkiSpZybnkiRJkiT1zORckiRJkqSemZxLkiRJktQz\nk3NJkiRJknqWqlp64+RiYJ9tF44maB/gp30HIY3BOatZ45zVrHHOatY4ZzVrVlfVM7e281jJuWZH\nkmuq6si+45CWyjmrWeOc1axxzmrWOGc1a5Y7Z13WLkmSJElSz0zOJUmSJEnqmcn5/18f7zsAaUzO\nWc0a56xmjXNWs8Y5q1mzrDnrPeeSJEmSJPXMK+eSJEmSJPXM5HwGJVmd5Ooka5PclOQDbf+BSa5K\n8r0k5yd5TNu/cyt/r9Uf0Gf82nElWZHk+iQXtbJzVlMryQ+TrEtyQ5Jr2r69klya5Lvt955tf5Kc\n3ebst5Mc0W/02hEl2SPJl5LcmuSWJM9zzmpaJXl6e36d+7k/yTuds5pmSd7V8q8bk5zX8rKJvZ41\nOZ9NDwHHVdWzgDXAiUmOBj4CnFlVBwP3AW9u7d8M3Nf2n9naSX04HbhloOyc1bQ7tqrWDHwtynuA\nr1XV04CvtTLAS4GntZ+3Auds90glOAu4uKoOAZ5F93zrnNVUqqrvtOfXNcBzgAeBL+Oc1ZRKsh/w\nDuDI9l3mK4BTmeDrWZPzGVSdB1pxVfsp4DjgS23/54BXtu1XtDKt/vgk2U7hSgAk2R94OfDJVg7O\nWc2ewbk5PGc/356frwT2SLJvHwFqx5Rkd+AFwKcAqupXVfUznLOaDccD36+q23DOarqtBB6bZCWw\nC3AnE3w9a3I+o9ry4BuAu4FLge8DP6uqja3Jj4D92vZ+wB0ArX49sPf2jVjio8AfAw+38t44ZzXd\nCvhqkmuTvLXte1JV3dm2fwI8qW0/MmebwfksbQ8HAvcAn2m3D30yya44ZzUbTgXOa9vOWU2lqvox\n8FfA7XRJ+XrgWib4etbkfEZV1aa2DGh/4LnAIT2HJC0oyUnA3VV1bd+xSGN4flUdQbeU8u1JXjBY\nWd3XnfiVJ5oWK4EjgHOq6tnAL3h0OTDgnNV0avfnngx8cbjOOatp0j7/4BV0b4Y+GdgVOHGSxzA5\nn3FtydrlwPPolvesbFX7Az9u2z8GngLQ6ncH/nc7h6od2zHAyUl+CPwD3fKfs3DOaoq1d8ipqrvp\n7oN8LnDX3DLK9vvu1vyROdsMzmdpe/gR8KOquqqVv0SXrDtnNe1eClxXVXe1snNW0+oE4L+r6p6q\n2gD8I91r3Im9njU5n0FJnpBkj7b9WODFdB/6cjnwqtbsDcBX2vaFrUyrv6z8gnttR1X1J1W1f1Ud\nQLd07bKqei3OWU2pJLsmefzcNvAS4EY2n5vDc/b17dOEjwbWDyzLlLa5qvoJcEeSp7ddxwM345zV\n9DuNR5e0g3NW0+t24Ogku7R7x+eeZyf2eja+3p09SQ6n+3CBFXRvsFxQVWckOYjuquRewPXA66rq\noSSrgXOBZwP3AqdW1Q/6iV47uiQvAt5dVSc5ZzWt2tz8ciuuBL5QVR9KsjdwAfBU4DbgNVV1b/tP\n+mN0y9seBN5UVdf0ELp2YEnW0H3o5mOAHwBvor1OwDmrKdTe/LwdOKiq1rd9Ps9qaqX7CutTgI10\nr11/j+7e8om8njU5lyRJkiSpZy5rlyRJkiSpZybnkiRJkiT1zORckiRJkqSemZxLkiRJktQzk3NJ\nkiRJknpmci5JmhpJNiW5YeDnPW3/15McuRXjrUnyshH1RyY5eytjnTemJKuSfDjJd5Ncl+SbSV66\nNceYZ+wHtrLfZuchyclz53ZEnzOSnNC235lklzGPmSSXJdltkXZHJ7mq/b1vSfL+gbpVSa4b57gL\nHOPfkuy53HEkSdqWVvYdgCRJA35ZVWsmON4a4EjgX4Yrkqxs35E76e/J/QtgX+CZ7XtOnwS8cMLH\nGNdm56GqLgQuHNWhqt43UHwn8Hd03y28VC8D1lbV/Yu0+xzd9xivTbICePpA3fOBK8Y45kLOBf4A\n+NAExpIkaZvwyrkkaaYkeUm7Gn1dki8meVzbf1SS/0yyNsnVSXYHzgBOaVdlT0ny/iTnJrkCODfJ\ni5Jc1Po/LslnkqxL8u0kv9X2n5PkmiQ3JfnAIrHtArwF+MOqegigqu6qqgta/Wlt/BuTfGSg3wNJ\nPtRiv7Il9CQ5sD3WdUk+OND+kbhb+WNJ3jjGeXhj67N7ktuS7NT67prkjnbF+rNJXpXkHcCTgcuT\nXJ7kd5N8dODYb0ly5jyn47XAV1qbA5LcONDn3QNXyJ8I3NnO1aaqunlgjBOBf219Xt/+LmuTnNv2\nfbb9fa5M8oN2Xj7drsB/dmCcC4HTRv3tJEnqm8m5JGmaPDabL2s/ZbAyyT7Ae4ETquoIuqvef5Tk\nMcD5wOlV9SzgBOAXwPuA86tqTVWd34Y5tPUfTtb+HFhfVYdV1eHAZW3/n1XVkcDhwAuTHD4i/oOB\n2+e7WpzkycBHgOPormQfleSVrXpX4MoW+7/TJfgAZwHnVNVhtAR2lDHPA1W1HriBR6/snwRcUlUb\nBtqcDfwPcGxVHQtcAPxmklWtyZuAT88TzjHAtYvFDJwJfCfJl5O8Lcnqgbpjga8neQbd3/249rhO\nH2izJ/A84F10SfiZwDOAw5KsaY/hPmDnJHsvIR5Jknphci5Jmia/bAnkmuFEsjmaLrm+IskNwBuA\nX6dbCn1nVX0LoKrur6qNCxzjwqr65Tz7TwD+dq7QEjqA17T7nq+nS/oO3crHdhTw9aq6p8X298AL\nWt2vgLkr4dcCB7TtY4Dz2va5SzjGOOdhzvnA3Jsgp7bygqrqAbo3Lk5KcgiwqqrWzdN0r6r6+WIB\nV9UZdEvuvwr8NnAxQJL9gHur6kG6NzS+WFU/bX3uHRjin6qqgHXAXVW1rqoeBm7i0fMIcDfdCgBJ\nkqaS95xLkmZJgEuHr3onOWyMMX6x5IMlBwLvBo6qqvvaUunVI7p8D3hqkt2WcK/1oA0twQTYxOb/\nP9c87Tey+Rvso2JazIXAXybZC3gOj64YGOWTwJ8CtwKfWaDNxiQ7tUR5ZLxV9X3gnCSfAO5pV7hP\nBC5ZQiwPtd8PD2zPlQfP42pgvjdlJEmaCl45lyTNkiuBY5IcDI/cI/0bwHeAfZMc1fY/PslK4OfA\n45c49qXA2+cK7dO9d6NL5te3+8BHfup6u8r7KeCstsScJE9I8mrgarpl8fu0Dz47DfjGIjFdQXc1\nG7p7uOfcBhyaZOckewDHt/1jn4d2JfxbdEvoL6qqTfM026x/VV0FPIXuSvd587Sfi+Wgtn0X8MQk\neyfZmW75PC3GlydJKz6N7s2JnzFwvzndGwavnluW3t5IWLI2/q8BPxynnyRJ25PJuSRpmgzfc/7h\nwcqqugd4I3Bekm8D3wQOqapf0S3N/pska+kS7dXA5XRJ7Bb3r8/jg8Ce7cPa1tLdY72Wbjn7rcAX\nWNonh78XuAe4uX0I2kXA/VV1J/CeFtNa4Nqq+soiY50OvD3JOmC/gfNwB9293ze239e3/Vt7Hs4H\nXsfCS9o/Dlyc5PKBfRcAVwws/x/2z8CLWlwb6D6U7uoW060D7X6H7p7zG+iW7s+9CXFwVd3a+t9E\n90nr32iP668XOOZCnkN3T/9iS/wlSepNHl1FJ0mStDTt0+LPrKqvLVC/L/D5qnrxVoz9fOB1VfX7\nywxzbryz6D5rYN5YJUmaBl45lyRJS5ZkjyT/RffhfQsmu22lwCeS7DbuMarqPyaVmDc3mphLkqad\nV84lSZIkSeqZV84lSZIkSeqZybkkSZIkST0zOZckSZIkqWcm55IkSZIk9czkXJIkSZKknpmcS5Ik\nSZLUs/8D8gLjmV3DLV8AAAAASUVORK5CYII=\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "5a1e07d47f5646449dfda8110c5f39b7" - } - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "\n", "# Create a map centered on this data log.\n", "center = [pose['latitude'].median(), pose['longitude'].median()]\n", "# print center\n", "zoom = 17\n", - "m = Map(center=center, zoom=zoom, height='800px')\n", + "m = Map(center=center, zoom=zoom, height='1000px')\n", "if sensor_name in data:\n", " m += io # Add image overlay\n", "if sensor_name not in data:\n", @@ -297,9 +289,9 @@ "\n", "if sensor_name in data:\n", " # Set the colormap and norm to correspond to the data for which\n", - " # the colorbar will be used.\n", + " # the colorbar will be used. \n", " cmap = matplotlib.cm.jet\n", - " norm = matplotlib.colors.Normalize(vmin=data_min, vmax=data_max)\n", + " norm = matplotlib.colors.Normalize(vmin=data_min, vmax=NORMALIZER)\n", " cb1 = matplotlib.colorbar.ColorbarBase(ax1, cmap=cmap,\n", " norm=norm,\n", " orientation='horizontal')\n", @@ -313,7 +305,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [] diff --git a/src/platypus/io/logs.py b/src/platypus/io/logs.py index cda25ac..96304c4 100644 --- a/src/platypus/io/logs.py +++ b/src/platypus/io/logs.py @@ -105,9 +105,10 @@ def merge_files(filename_list): logfile_result_list = [load(filename) for filename in filename_list] if len(logfile_result_list) == 1: return logfile_result_list[0] - all_data_types = set() - for i in range(1, len(logfile_result_list)): - all_data_types = all_data_types.union(set(logfile_result_list[i].keys())) + #all_data_types = set() + #for i in range(1, len(logfile_result_list)): + # all_data_types = all_data_types.union(set(logfile_result_list[i].keys())) + all_data_types = {key for log_dict in logfile_result_list for key in log_dict.keys()} print all_data_types merged_dataframe_dict = dict() From 492d656f5e281e31ba4427698cbbed0b3adf727e Mon Sep 17 00:00:00 2001 From: jjblum Date: Mon, 27 Nov 2017 18:00:48 +0100 Subject: [PATCH 22/43] Need to debug why certain files cause infinite memory usage issues. --- notebooks/Data_Interpolation.ipynb | 60 ++++++++++++++++++++++-------- src/platypus/io/logs.py | 10 +++-- 2 files changed, 50 insertions(+), 20 deletions(-) diff --git a/notebooks/Data_Interpolation.ipynb b/notebooks/Data_Interpolation.ipynb index 2e4bab7..d7673b6 100644 --- a/notebooks/Data_Interpolation.ipynb +++ b/notebooks/Data_Interpolation.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 43, "metadata": { "collapsed": false, "deletable": true, @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 44, "metadata": { "collapsed": false, "deletable": true, @@ -40,9 +40,11 @@ "name": "stdout", "output_type": "stream", "text": [ + "set([u'ATLAS_PH', u'BATTERY', 'pose', u'ES2'])\n", "ES2 sensor is present. Trimming all data within EC = 0 time windows\n", "\n", "Available sensors/channels:\n", + " ATLAS_PH, ph\n", " ES2, ec\n", " ES2, temperature\n" ] @@ -50,9 +52,18 @@ ], "source": [ "# Import the data from the specified logfile\n", - "log_path = \"/home/jason/Documents/INTCATCH/phone logs/fishing pond/2017-10-12/\"\n", - "log_filenames = [log_path + \"platypus_20171012_072334.txt\"]\n", - "csv_output_filename = \"2017_10_12__Atlantide\"\n", + "log_path = \"/home/jason/Documents/Platypus/phone logs/netherlands/Nov2017Shawn/netherlands day two/\"\n", + "log_filenames = [\n", + " log_path + \"platypus_20171119_081603.txt\",\n", + " log_path + \"platypus_20171119_093819.txt\",\n", + " #####log_path + \"platypus_20171119_032804.txt\"\n", + " log_path + \"platypus_20171119_105709.txt\",\n", + " #####log_path + \"platypus_20171119_035612.txt\",\n", + " #####log_path + \"platypus_20171119_031056.txt\",\n", + " log_path + \"platypus_20171119_102933.txt\",\n", + " log_path + \"platypus_20171119_112232.txt\"\n", + " ]\n", + "csv_output_filename = \"Nov2017ShawnNetherlands\"\n", "log_ext = \".txt\"\n", "\n", "#data = platypus.io.logs.load(log_path + log_filename + log_ext)\n", @@ -123,7 +134,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 45, "metadata": { "collapsed": true, "deletable": true, @@ -142,7 +153,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 46, "metadata": { "collapsed": false, "deletable": true, @@ -176,7 +187,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 47, "metadata": { "collapsed": false, "deletable": true, @@ -192,7 +203,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 48, "metadata": { "collapsed": false, "deletable": true, @@ -200,13 +211,10 @@ }, "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "/usr/local/lib/python2.7/dist-packages/numpy/core/fromnumeric.py:2909: RuntimeWarning: Mean of empty slice.\n", - " out=out, **kwargs)\n", - "/usr/local/lib/python2.7/dist-packages/numpy/core/_methods.py:73: RuntimeWarning: invalid value encountered in true_divide\n", - " ret, rcount, out=ret, casting='unsafe', subok=False)\n" + "Data min = 241.000000 Data max = 1850.500000\n" ] } ], @@ -264,13 +272,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 49, "metadata": { "collapsed": false, "deletable": true, "editable": true }, - "outputs": [], + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+EAAABRCAYAAAC9m6cOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAECRJREFUeJzt3XmQZWV5x/HvjxkWQdkkKgIJJKAWCE4QDBZGBQmiErEq\nLhCNoJZRYxRJTAJqXFBTqKkgJoYqF0SJQZDSOMEIEkGtEAdkGxZFIS4sIktAXDDADE/+OG/DnTu9\nzJ3uOd19+X6qpua8513ue8/bb/d9znnPuakqJEmSJEnShrfRfHdAkiRJkqSHC4NwSZIkSZJ6YhAu\nSZIkSVJPDMIlSZIkSeqJQbgkSZIkST0xCJckSZIkqScG4ZIkSZIk9cQgXJIkSZKknhiES5IkSZLU\nk6WjFE52LbhnIjWcO016lLJz3dawh1u/5tBcHp7ZtL1Y2ppN27a1YadMb23VDHXXzM805Yfzpivb\nJWvS7ckM509Xfu284bozlV+/1x21/GzaXij9GLXtuWzr4XC8Rmt7/X/Oh/Pn9T3VQD+mn8pr5w+b\n+vCsnZ6u7kzlZyq7oeoO58/le/R4jfbaG7KtUeqOWnYO666VXZNvT5qephvD+aO+pdkM24Y81Buy\nXzP9ON0C51bVIZNkrWGkILwLwF83RdWNp2l6urxR82eqO5dtzdd7nMu2RjT4yWLJiC87nB6sP5u6\nw/VnU3e4/GzqDqcX6/HakG1Nl57LYz/bfs3m2K+VHviVvHT1UN6a6Y2WrFoze+M185cMlF+ydDhv\nqO5w/kYDdRnKWys91NY05WdTd8O2tf51h+uPXncu25r6fcymH6O3NZufiYXy8zXbn4mF9/M1cj9W\nD6VXDfRj9QNDeUybztCvszW6smqaPIDp6g6nRyk702vZr/nr12zqPhz6NWLd+4fyB6Yy96+eOm/S\nukMvdf8U2zOVnSl/NnWH84fz5rKtuXyPAO+G7SbZvRaXo0uSJEmS1BODcEmSJEmSemIQLkmSJElS\nTwzCJUmSJEnqiUG4JEmSJEk9MQiXJEmSJKknBuGSJEmSJPXEIFySJEmSpJ4YhEuSJEmS1BODcEmS\nJEmSemIQLkmSJElSTwzCJUmSJEnqiUG4JEmSJEk9MQiXJEmSJKknBuGSJEmSJPXEIFySJEmSpJ4Y\nhEuSJEmS1BODcEmSJEmSemIQLkmSJElSTwzCJUmSJEnqiUG4JEmSJEk9MQiXJEmSJKknBuGSJEmS\nJPXEIFySJEmSpJ4YhEuSJEmS1BODcEmSJEmSemIQLkmSJElSTwzCJUmSJEnqiUG4JEmSJEk9MQiX\nJEmSJKknBuGSJEmSJPUkVbXuhZNzgO02XHc0g+2AO+a7E9qgHOPx5viOP8d4/DnG488xHm+O7/ib\nzzG+o6oOmanQSEG45leSS6pqn/nuhzYcx3i8Ob7jzzEef47x+HOMx5vjO/4Wwxi7HF2SJEmSpJ4Y\nhEuSJEmS1BOD8MXlY/PdAW1wjvF4c3zHn2M8/hzj8ecYjzfHd/wt+DH2nnBJkiRJknrilXBJkiRJ\nknpiEL7AJFmS5PIkZ7f0LkkuSnJ9kjOSbNL2b9rS17f8neez31o3SbZOclaSa5N8N8nTk2yb5Lwk\n17X/t2llk+QjbYyvTLL3fPdfM0tyTJJrklyd5PQkmzmPF7ckpyS5LcnVA/tGnrdJjmzlr0ty5Hy8\nF61tivH9UPs9fWWSLybZeiDvuDa+30vy3IH9h7R91yc5tu/3oalNNsYDeX+ZpJJs19LO4UVoqjFO\n8qY2l69J8sGB/c7jRWaK39XLkqxIckWSS5I8re1f8PPYIHzhORr47kD6A8CJVbUrcBfwmrb/NcBd\nbf+JrZwWvpOAc6rqScBT6Mb6WOBrVbUb8LWWBngesFv796fAyf13V6NIsgPwZmCfqnoysAQ4HOfx\nYncqMPydnyPN2yTbAu8Cfg94GvCuicBd8+5U1h7f84AnV9VewPeB4wCS7E43p/dodf453cnzJcBH\n6cZ/d+CIVlYLw6msPcYk2Qk4GLhhYLdzeHE6laExTnIAcBjwlKraA/j7tt95vDidytrz+IPAe6pq\nGfDOloZFMI8NwheQJDsCLwA+0dIBDgTOakU+DbyobR/W0rT857TyWqCSbAU8E/gkQFXdV1U/Y82x\nHB7jz1RnBbB1ku177rZGtxR4RJKlwObALTiPF7Wq+iZw59DuUeftc4HzqurOqrqLLshbKyhQ/yYb\n36r6alWtaskVwI5t+zDgc1V1b1X9ELie7oPc04Drq+oHVXUf8LlWVgvAFHMYupOffw0MPiDJObwI\nTTHGbwBOqKp7W5nb2n7n8SI0xRgXsGXb3gr4Sdte8PPYIHxh+TDdH4MHWvrRwM8GPgjcBOzQtncA\nbgRo+Xe38lq4dgFuBz6V7paDTyTZAnhsVd3SyvwUeGzbfnCMm8Hx1wJUVTfTnWm/gS74vhu4FOfx\nOBp13jqfF69XA19p247vmEhyGHBzVa0cynKMx8cTgN9vt3t9I8m+bb9jPD7eAnwoyY10n7+Oa/sX\n/BgbhC8QSQ4FbquqS+e7L9pglgJ7AydX1e8Cv+KhJawAVPd1BX5lwSLVljQdRnfC5fHAFnilZOw5\nb8dXkrcDq4DPzndfNHeSbA68jW75qsbXUmBbYD/gr4AzXW02dt4AHFNVOwHH0FabLgYG4QvH/sAL\nk/yIbvnLgXT3D2/dlrVCtxzu5rZ9M7ATQMvfCvjfPjuskd0E3FRVF7X0WXRB+a0Ty8zb/xPLpR4c\n42Zw/LUwHQT8sKpur6r7gS/QzW3n8fgZdd46nxeZJEcBhwIvr4e+z9XxHQ+/Q3eydGX73LUjcFmS\nx+EYj5ObgC+0JckX06003Q7HeJwcSfdZC+DzdLcUwCIYY4PwBaKqjquqHatqZ7qHRZxfVS8HLgBe\n3IodCXypbS9vaVr++QMfErQAVdVPgRuTPLHteg7wHdYcy+ExfmV7wuN+wN0Dy1+1MN0A7Jdk83a2\nfWKMncfjZ9R5ey5wcJJt2oqJg9s+LUBJDqG7PeyFVXXPQNZy4PB032ywC91Dfy4Gvg3slu6bEDah\n+zu+vO9+a91U1VVV9Ziq2rl97roJ2Lv9nXYOj49/Aw4ASPIEYBPgDpzH4+QnwLPa9oHAdW17wc/j\npTMX0Tz7G+BzSd4HXM5Dyyw+CZyW5Hq6hxQcPk/902jeBHy2/XL/AfAqupNhZyZ5DfBj4KWt7H8A\nz6d7YMg9rawWsKq6KMlZwGV0S1gvBz4GfBnn8aKV5HTg2cB2SW6ie7LqCYwwb6vqziTvpfuQB3B8\nVU32oCj1bIrxPQ7YFDivrV5dUVWvr6prkpxJd3JtFfDGqlrd2vlzug9zS4BTquqa3t+MJjXZGFfV\nVMtWncOL0BTz+BTglHRfaXUfcGQ70e08XoSmGOPXAie11YT/R/ckdFgE8zhedJEkSZIkqR8uR5ck\nSZIkqScG4ZIkSZIk9cQgXJIkSZKknhiES5IkSZLUE4NwSZIkSZJ6YhAuSVowkqxOcsXAv2Pb/q8n\n2Wc92luW5PnT5O+T5CPr2ddJ+5Rk4yQnJLkuyWVJvpXkeevzGpO0/cv1rLfGcUjywoljO02d45Mc\n1LbfkmTzEV8zSc5PsuUM5fZLclEb7+8mefdA3sZJLhvldad4jf9s3wkrSdK883vCJUkLya+ratkc\ntrcM2IfuO0PXkGRpVV0CXDKHrwfwXmB74MlVdW+SxwLPmuPXGNUax6GqlgPLp6tQVe8cSL4F+Be6\n71tdV88HVlbVz2co92ngpVW1MskS4IkDec8ALhzhNadyGvBnwPvnoC1JkmbFK+GSpEUlycHt6vJl\nST6f5JFt/75J/jvJyiQXJ9kKOB54WbvK+rIk705yWpILgdOSPDvJ2a3+I5N8KslVSa5M8kdt/8lJ\nLklyTZL3zNC3zYHXAm+qqnsBqurWqjqz5R/R2r86yQcG6v0yyftb31e0wJ0ku7T3elWS9w2Uf7Df\nLf1PSY4a4Tgc1epsleTHSTZqdbdIcmO7An1qkhcneTPweOCCJBckeXWSDw+89muTnDjJ4Xg58KVW\nZuckVw/UeevAFe/HALe0Y7W6qr4z0MYhwFdanVe2cVmZ5LS279Q2PiuS/KAdl1PaFfVTB9pZDhwx\n3dhJktQXg3BJ0kLyiKy5HP1lg5lJtgPeARxUVXvTXcX+iySbAGcAR1fVU4CDgF8B7wTOqKplVXVG\na2b3Vn84KPtb4O6q2rOq9gLOb/vfXlX7AHsBz0qy1zT93xW4YbKrv0keD3wAOJDuyvS+SV7UsrcA\nVrS+f5MukAc4CTi5qvakBarTGfE4UFV3A1fw0JX6Q4Fzq+r+gTIfAX4CHFBVBwBnAn+YZONW5FXA\nKZN0Z3/g0pn6DJwIfC/JF5O8LslmA3kHAF9PsgfduB/Y3tfRA2W2AZ4OHEMXbJ8I7AHsmWRZew93\nAZsmefQ69EeSpA3KIFyStJD8ugWKy4YDxmY/uiD6wiRXAEcCv0W3hPmWqvo2QFX9vKpWTfEay6vq\n15PsPwj46ESiBW4AL233JV9OF9ztvp7vbV/g61V1e+vbZ4Fntrz7gIkr25cCO7ft/YHT2/Zp6/Aa\noxyHCWcAEyc7Dm/pKVXVL+lOUBya5EnAxlV11SRFt62qX8zU4ao6nm6p/FeBPwbOAUiyA3BnVd1D\nd+Li81V1R6tz50AT/15VBVwF3FpVV1XVA8A1PHQcAW6ju6IvSdK88p5wSdJiEuC84avYSfYcoY1f\nrfOLJbsAbwX2raq72hLnzaapcj3wm0m2XId7oQfd3wJJgNWs+fe5Jim/ijVPpE/Xp5ksB/4uybbA\nU3loBcB0PgG8DbgW+NQUZVYl2agFxNP2t6r+Bzg5yceB29sV60OAc9ehL/e2/x8Y2J5IDx7HzYDJ\nTr5IktQrr4RLkhaTFcD+SXaFB+9hfgLwPWD7JPu2/Y9KshT4BfCodWz7POCNE4n2NO0t6YL2u9t9\n2tM+5bxdtf0kcFJbGk6S30jyEuBiuuXs27UHkB0BfGOGPl1Id3UaunusJ/wY2D3Jpkm2Bp7T9o98\nHNqV7W/TLX0/u6pWT1JsjfpVdRGwE92V69MnKT/Rl99u27cCj0ny6CSb0i17p/XxBUnSkrvRnYT4\nGQP3g9OdGHjJxHLydsJgnbX2Hwf8aJR6kiRtCAbhkqSFZPie8BMGM6vqduAo4PQkVwLfAp5UVffR\nLan+xyQr6QLqzYAL6ILVte4vn8T7gG3aQ9NW0t0DvZJuGfq1wL+ybk/qfgdwO/Cd9jCys4GfV9Ut\nwLGtTyuBS6vqSzO0dTTwxiRXATsMHIcb6e7Nvrr9f3nbv77H4QzgFUy9FP1jwDlJLhjYdyZw4cCy\n/WFfBp7d+nU/3cPhLm59unag3J/Q3RN+Bd2S+4mTDbtW1bWt/jV0Tzb/Rntf/zDFa07lqXT33M+0\nNF+SpA0uD61+kyRJWjft6ewnVtXXpsjfHvhMVf3BerT9DOAVVfX6WXZzor2T6J4FMGlfJUnqk1fC\nJUnSOkuydZLv0z1Eb8qgtl35/3iSLUd9jar6r7kKwJurDcAlSQuFV8IlSZIkSeqJV8IlSZIkSeqJ\nQbgkSZIkST0xCJckSZIkqScG4ZIkSZIk9cQgXJIkSZKknhiES5IkSZLUk/8Ht8qyghMh6V0AAAAA\nSUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c6b7440ca5544c89bdf73d6c78e00d70" + } + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "\n", "# Create a map centered on this data log.\n", diff --git a/src/platypus/io/logs.py b/src/platypus/io/logs.py index 96304c4..66aa1a7 100644 --- a/src/platypus/io/logs.py +++ b/src/platypus/io/logs.py @@ -185,10 +185,12 @@ def read_v4_2_0(logfile): v['p'][2], zone, hemi ]) - elif k == 'sensor': - raw_data[v['type']].append( - [timestamp] + v['data'] - ) + elif k == 'sensor': + try: + raw_data[v['type']].append([timestamp] + v['data']) + except: + # do nothing + None else: pass From de4e2144025e2f8383e74273c490be2cdb59d4f4 Mon Sep 17 00:00:00 2001 From: Shawn Hanna Date: Thu, 7 Dec 2017 07:43:52 +0000 Subject: [PATCH 23/43] saving netherlands scripts --- examples/EC_zero_trim.py | 4 +- examples/histogram.py | 92 ++++++++++++ examples/loading_logs.py | 87 +++++++++-- notebooks/Data_Interpolation.ipynb | 230 ++++++++++++++++++++--------- 4 files changed, 328 insertions(+), 85 deletions(-) create mode 100644 examples/histogram.py diff --git a/examples/EC_zero_trim.py b/examples/EC_zero_trim.py index 7185e4a..2cda1a8 100644 --- a/examples/EC_zero_trim.py +++ b/examples/EC_zero_trim.py @@ -7,6 +7,8 @@ import six import re import pandas +import glob + # FILE TO TEST JAR DATA EXTRACTION PATH = "/home/jason/Documents/INTCATCH/phone logs/Gardaland outlet/2017-10-3/" @@ -188,7 +190,7 @@ def extract_sampler_data_by_jar(): if __name__ == "__main__": - merged_data = merge_files([PATH2 + FILE1 + EXT, PATH2 + FILE2 + EXT]) + merged_data = merge_files(glob.glob("/home/shawn/day2/*.txt")) diff --git a/examples/histogram.py b/examples/histogram.py new file mode 100644 index 0000000..ee271e8 --- /dev/null +++ b/examples/histogram.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# coding: utf-8 + +""" +Example of loading Platypus vehicle logs from file. + +Data is loaded as time series into Pandas[1] DataFrames, from which they can +be interpolated and filtered for other purposes, or used as input into Numpy[2] +or Scipy[3] functions. + +[1]: http://pandas.pydata.org/ +[2]: http://www.numpy.org/ +[3]: http://www.scipy.org/ +""" +import matplotlib.pyplot as plt +import platypus.io.logs +import platypus.util.conversions +import glob +import os +import numpy as np +import plotly.plotly as py + +# Read the data log from file. +# Note: for log versions <5.0, this filename must be 'airboat_[timestamp].txt]. + +def trim_using_EC(dataframe, threshold=100): + """ + Trims any data when EC < 100 + :return: trimmed dataframe + """ + if "ES2" in dataframe: + print "ES2 sensor is present. Trimming all data within EC < {:.0f} time windows\n".format(threshold) + # find all time windows where EC is exactly 0 + ES2_data = dataframe["ES2"] + values = ES2_data["ec"].values + ec_eq_zero_indices = np.where(values < threshold)[0] + windows = list() + windows.append([ec_eq_zero_indices[0]]) + left = ec_eq_zero_indices[0] + for ii in range(1, ec_eq_zero_indices.shape[0]): + i = ec_eq_zero_indices[ii] + if i - left > 5: + # there has been a jump in index, a new time window has started + windows[-1].append(left) + windows.append([i]) + left = i + windows[-1].append(ec_eq_zero_indices[-1]) + # print ec_eq_zero_indices + # print windows + for window in windows: + time_window = [ES2_data["ec"].index.values[window[0]], ES2_data["ec"].index.values[window[1]]] + for k in dataframe: + dataframe[k] = dataframe[k].loc[np.logical_or(dataframe[k].index < time_window[0], dataframe[k].index > time_window[1])] + else: + print "No ES2 sensor present. No trimming will be performed." + return dataframe + + +folders = ['/home/shawn/day4','/home/shawn/day3','/home/shawn/day2','/home/shawn/day1'] + +for folder in folders: + data = platypus.io.logs.merge_files(glob.glob(folder+'/*.txt')) + data = trim_using_EC(data, 200) + + channel = 'ph' + sensor ='ATLAS_PH' + + # Get the standard deviation of the ES2 data. + es2_stddev = data[sensor][channel].std() + print channel +" std", es2_stddev + + # Get the mean of the ES2 data. + es2_mean = data[sensor][channel].mean() + print channel+" mean", es2_mean + + # bins = np.arange(6,13,0.5) + # bins = np.arange(200,1800,100) + bins = np.arange(5.5, 9 + 0.5, 0.25) + # n, bins, patches = plt.hist(data[sensor][channel], bins=xrange(200,1600,100)) + n, bins, patches = plt.hist(data[sensor][channel], normed=False, bins=bins) + + print n, bins, patches + + plt.xlabel(channel) + plt.ylabel('Counts') + plt.title('Histogram of ' + channel + ' ' + folder.split('/')[-1]) + plt.savefig('Histogram of ' + channel + ' ' + folder.split('/')[-1]+'.png') + # plt.text(0, .25, "Standard Dev: " + str(es2_stddev)) + plt.figtext(.16, .75, "Mean: " + str(es2_mean)) + plt.figtext(.16, .7, "std: " + str(es2_stddev)) + plt.grid(True) + plt.show() diff --git a/examples/loading_logs.py b/examples/loading_logs.py index 31fdfe8..e91ef84 100644 --- a/examples/loading_logs.py +++ b/examples/loading_logs.py @@ -15,26 +15,87 @@ import matplotlib.pyplot as plt import platypus.io.logs import platypus.util.conversions +import glob +import os +import numpy as np # Read the data log from file. # Note: for log versions <5.0, this filename must be 'airboat_[timestamp].txt]. -data = platypus.io.logs.load('/home/jason/Documents/INTCATCH/phone logs/fishing pond/platypus_20170307_053438.txt') +def trim_using_EC(dataframe, threshold=100): + """ + Trims any data when EC < 100 + :return: trimmed dataframe + """ + if "ES2" in dataframe: + print "ES2 sensor is present. Trimming all data within EC < {:.0f} time windows\n".format(threshold) + # find all time windows where EC is exactly 0 + ES2_data = dataframe["ES2"] + values = ES2_data["ec"].values + ec_eq_zero_indices = np.where(values < threshold)[0] + windows = list() + windows.append([ec_eq_zero_indices[0]]) + left = ec_eq_zero_indices[0] + for ii in range(1, ec_eq_zero_indices.shape[0]): + i = ec_eq_zero_indices[ii] + if i - left > 5: + # there has been a jump in index, a new time window has started + windows[-1].append(left) + windows.append([i]) + left = i + windows[-1].append(ec_eq_zero_indices[-1]) + # print ec_eq_zero_indices + # print windows + for window in windows: + time_window = [ES2_data["ec"].index.values[window[0]], ES2_data["ec"].index.values[window[1]]] + for k in dataframe: + dataframe[k] = dataframe[k].loc[np.logical_or(dataframe[k].index < time_window[0], dataframe[k].index > time_window[1])] + else: + print "No ES2 sensor present. No trimming will be performed." + return dataframe -# Access the first 100 GPS locations for the vehicle. -poses = data['pose'] -# Plot the first 100 GPS locations as UTM coordinates using matplotlib. -plt.plot(poses['easting'], poses['northing']) -plt.show() +sensor = "ES2" +channel = 'temperature' -# Retrieve temperature data from the ES2 sensor. -temp = data['ES2']['temp'] +for folder in glob.glob('/home/shawn/day*/'): + print "processing folder: ", folder + for file in glob.glob(folder+'*.txt'): + print file + out_name = os.path.basename(os.path.splitext(file)[0]) + print out_name + data = platypus.io.logs.merge_files(glob.glob(folder+"/*.txt")) + data = trim_using_EC(data, 200) -# Plot ES2 electrical conductivity data using matplotlib. -plt.plot(data['ES2'].index, data['ES2']['ec']) -plt.show() + # Access the first 100 GPS locations for the vehicle. + poses = data['pose'] -# Get the standard deviation of the ES2 data. -es2_stddev = data['ES2'].std() + output_base = folder+out_name+"_"+channel + + plt.title("Path of vehicle: " + out_name) + # Plot the first 100 GPS locations as UTM coordinates using matplotlib. + plt.plot(poses['easting'], poses['northing']) + plt.savefig(output_base + "_path.png") + # plt.show() + plt.clear() + + # Retrieve temperature data from the ES2 sensor. + # temp = data[sensor]['temp'] + + + # Plot ES2 electrical conductivity data using matplotlib. + + # print data[sensor][channel] + + plt.title("Graph of "+channel+" data: " + out_name) + plt.plot(data[sensor].index, data[sensor][channel]) + # plt.savefig(folder+"_"+channel+"_graph.png") + plt.savefig(output_base + "_graph.png") + + # plt.show() + plt.clear() + + # Get the standard deviation of the ES2 data. + es2_stddev = data[sensor].std() + print "deviation", es2_stddev diff --git a/notebooks/Data_Interpolation.ipynb b/notebooks/Data_Interpolation.ipynb index d7673b6..6c53beb 100644 --- a/notebooks/Data_Interpolation.ipynb +++ b/notebooks/Data_Interpolation.ipynb @@ -2,12 +2,8 @@ "cells": [ { "cell_type": "code", - "execution_count": 43, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 1, + "metadata": {}, "outputs": [], "source": [ "%matplotlib inline\n", @@ -29,22 +25,21 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 21, "metadata": { - "collapsed": false, - "deletable": true, - "editable": true + "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "set([u'ATLAS_PH', u'BATTERY', 'pose', u'ES2'])\n", + "set([u'ATLAS_PH', u'BATTERY', 'pose', u'ATLAS_DO', u'ES2'])\n", "ES2 sensor is present. Trimming all data within EC = 0 time windows\n", "\n", "Available sensors/channels:\n", " ATLAS_PH, ph\n", + " ATLAS_DO, do\n", " ES2, ec\n", " ES2, temperature\n" ] @@ -52,17 +47,50 @@ ], "source": [ "# Import the data from the specified logfile\n", - "log_path = \"/home/jason/Documents/Platypus/phone logs/netherlands/Nov2017Shawn/netherlands day two/\"\n", + "# log_path = \"/home/shawn/day2/\"\n", + "# log_filenames = [\n", + "# log_path + \"platypus_20171119_081603.txt\",\n", + "# log_path + \"platypus_20171119_093819.txt\",\n", + "# log_path + \"platypus_20171119_102933.txt\",\n", + "# log_path + \"platypus_20171119_105652.txt\",\n", + "# log_path + \"platypus_20171119_105709.txt\",\n", + "# log_path + \"platypus_20171119_112232.txt\"\n", + "# ]\n", + "\n", + "log_path = \"/home/shawn/day1/\"\n", "log_filenames = [\n", - " log_path + \"platypus_20171119_081603.txt\",\n", - " log_path + \"platypus_20171119_093819.txt\",\n", - " #####log_path + \"platypus_20171119_032804.txt\"\n", - " log_path + \"platypus_20171119_105709.txt\",\n", - " #####log_path + \"platypus_20171119_035612.txt\",\n", - " #####log_path + \"platypus_20171119_031056.txt\",\n", - " log_path + \"platypus_20171119_102933.txt\",\n", - " log_path + \"platypus_20171119_112232.txt\"\n", - " ]\n", + " log_path + \"platypus_20171117_034941.txt\",\n", + "# log_path + \"platypus_20171117_045142.txt\",\n", + "# log_path + \"platypus_20171117_052611.txt\",\n", + "# log_path + \"platypus_20171117_063759.txt\",\n", + "# log_path + \"platypus_20171117_083516.txt\",\n", + "# log_path + \"platypus_20171117_085402.txt\",\n", + "# log_path + \"platypus_20171117_092146.txt\",\n", + "# log_path + \"platypus_20171117_095701.txt\",\n", + "# log_path + \"platypus_20171117_102132.txt\",\n", + "# log_path + \"platypus_20171117_102340.txt\",\n", + "# log_path + \"platypus_20171117_103741.txt\",\n", + " ]\n", + "\n", + "log_path = \"/home/shawn/day4/\"\n", + "log_filenames = [\n", + " log_path + \"platypus_20171121_045529.txt\",\n", + "# log_path + \"platypus_20171121_053515.txt\",\n", + "# log_path + \"platypus_20171121_073237.txt\",\n", + "# log_path + \"platypus_20171121_074700.txt\",\n", + "# log_path + \"platypus_20171121_084718.txt\",\n", + "# log_path + \"platypus_20171121_092302.txt\",\n", + "# log_path + \"platypus_20171121_095421.txt\",\n", + "# log_path + \"platypus_20171121_101351.txt\",\n", + "# log_path + \"platypus_20171121_103427.txt\",\n", + "# log_path + \"platypus_20171121_103725.txt\",\n", + "# log_path + \"platypus_20171121_110202.txt\",\n", + "# log_path + \"platypus_20171121_113356.txt\"\n", + " ]\n", + "\n", + "log_path = \"/home/shawn/all/\"\n", + "log_filenames = glob.glob(log_path+'/'+\"*.txt\")\n", + "\n", "csv_output_filename = \"Nov2017ShawnNetherlands\"\n", "log_ext = \".txt\"\n", "\n", @@ -75,7 +103,7 @@ " ES2_data = data[\"ES2\"]\n", " values = ES2_data[\"ec\"].values\n", " #ec_eq_zero_indices = np.where(values == 0)[0]\n", - " ec_eq_zero_indices = np.where(values < 200)[0]\n", + " ec_eq_zero_indices = np.where( (values < 200) )[0]\n", " windows = list()\n", " windows.append([ec_eq_zero_indices[0]])\n", " left = ec_eq_zero_indices[0]\n", @@ -102,7 +130,7 @@ " # find all time windows where pH is less than 6\n", " pH_data = data[\"ATLAS_PH\"]\n", " values = pH_data[\"ph\"].values\n", - " pH_lt_6_indices = np.where(values < 6)[0]\n", + " pH_lt_6_indices = np.where( (values < 6) | (values > 8.5))[0]\n", " windows = list()\n", " windows.append([pH_lt_6_indices[0]])\n", " left = pH_lt_6_indices[0]\n", @@ -134,30 +162,27 @@ }, { "cell_type": "code", - "execution_count": 45, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "execution_count": 28, + "metadata": {}, "outputs": [], "source": [ "# Select the sensor and the name of the channel for that sensor.\n", - "sensor_name = 'ES2'\n", - "sensor_channel = 'ec'\n", + "# sensor_name = 'ATLAS_PH'\n", + "# sensor_channel = 'ph'\n", + "# sensor_units = \"pH\"\n", + "\n", + "sensor_name = 'ATLAS_PH'\n", + "sensor_channel = 'ph'\n", "sensor_units = 'Electrical Conductivity (uS/cm)'\n", - "#sensor_units = 'Temperature (C)'\n", - "#sensor_units = 'Dissolved Oxygen (mg/L)'\n", - "#sensor_units = \"pH\"" + "\n", + "# sensor_units = 'Temperature (C)'\n", + "#sensor_units = 'Dissolved Oxygen (mg/L)'" ] }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 29, "metadata": { - "collapsed": false, - "deletable": true, - "editable": true, "scrolled": true }, "outputs": [], @@ -187,12 +212,8 @@ }, { "cell_type": "code", - "execution_count": 47, - "metadata": { - "collapsed": false, - "deletable": true, - "editable": true - }, + "execution_count": 24, + "metadata": {}, "outputs": [], "source": [ "# Create a trail of the vehicle's path on the map.\n", @@ -203,28 +224,49 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 25, "metadata": { - "collapsed": false, - "deletable": true, - "editable": true + "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Data min = 241.000000 Data max = 1850.500000\n" + "latitude 51.980245\n", + "longitude 4.258278\n", + "dtype: float64\n", + "latitude 51.994891\n", + "longitude 4.291409\n", + "dtype: float64\n", + "[1e-05, 1e-05]\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0mdata_estimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msensor\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'latitude'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'longitude'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msensor\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msensor_channel\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m \u001b[0mdata_zv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata_estimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_xy\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 29\u001b[0m \u001b[0mdata_zv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata_zv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/home/shawn/src/analytics/venv/local/lib/python2.7/site-packages/sklearn/neighbors/regression.pyc\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m 290\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mweights\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 291\u001b[0m y_pred = np.array([np.mean(_y[ind, :], axis=0)\n\u001b[0;32m--> 292\u001b[0;31m for ind in neigh_ind])\n\u001b[0m\u001b[1;32m 293\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 294\u001b[0m y_pred = np.array([(np.average(_y[ind, :], axis=0,\n", + "\u001b[0;32m/home/shawn/src/analytics/venv/local/lib/python2.7/site-packages/numpy/core/fromnumeric.pyc\u001b[0m in \u001b[0;36mmean\u001b[0;34m(a, axis, dtype, out, keepdims)\u001b[0m\n\u001b[1;32m 2907\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2908\u001b[0m return _methods._mean(a, axis=axis, dtype=dtype,\n\u001b[0;32m-> 2909\u001b[0;31m out=out, **kwargs)\n\u001b[0m\u001b[1;32m 2910\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2911\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/home/shawn/src/analytics/venv/local/lib/python2.7/site-packages/numpy/core/_methods.pyc\u001b[0m in \u001b[0;36m_mean\u001b[0;34m(a, axis, dtype, out, keepdims)\u001b[0m\n\u001b[1;32m 58\u001b[0m \u001b[0;31m# Make this warning show up first\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 59\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrcount\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 60\u001b[0;31m \u001b[0mwarnings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwarn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Mean of empty slice.\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mRuntimeWarning\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstacklevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 61\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[0;31m# Cast bool, unsigned int, and int to float64 by default\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], "source": [ - "# Add a data overlay for the map\n", + "## Add a data overlay for the map\n", "data_padding = [0.0001, 0.0001] # degrees lat/lon\n", "data_resolution = [0.00001, 0.00001] # degrees lat/lon\n", - "data_interpolation_radius = 0.00005 # degrees lat/lon\n", + "data_interpolation_radius = 0.0002 # degrees lat/lon\n", "data_bounds = [(position.min() - data_padding).tolist(),\n", " (position.max() + data_padding).tolist()]\n", + "print position.min()\n", + "print position.max()\n", + "print data_resolution\n", "\n", "# Create a rectangular grid of overlay points.\n", "data_xv, data_yv = np.meshgrid(\n", @@ -251,16 +293,46 @@ " data_min = data_zv[np.isfinite(data_zv)].min()\n", " print \"Data min = {:f} Data max = {:f}\".format(data_min, data_max)\n", " NORMALIZER = data_max # 800\n", - " data_zv = (data_zv - data_min) / (NORMALIZER - data_min)\n", + " data_zv = (data_zv - data_min) / (NORMALIZER - data_min)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "removing file: ./platypus_data_f5ac49ed-6ff4-49f2-b515-0c18c4a248cf.png\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/shawn/src/analytics/venv/lib/python2.7/site-packages/ipykernel_launcher.py:15: DeprecationWarning: `imsave` is deprecated!\n", + "`imsave` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n", + "Use ``imageio.imwrite`` instead.\n", + " from ipykernel import kernelapp as app\n" + ] + } + ], + "source": [ "\n", + "if sensor_name in data:\n", " # Update a color map only at the points that have valid values.\n", " data_rgb = np.zeros((data_shape[0], data_shape[1], 4), dtype=np.uint8)\n", - " data_rgb = matplotlib.cm.jet(data_zv)*255.0\n", + " data_rgb = matplotlib.cm.jet(data_zv) * 255\n", " data_rgb[:,:,3] = 255 * np.isfinite(data_zv)\n", "\n", " # Remove any old image files.\n", " old_png_files = glob.glob('./*.png')\n", " for old_png_file in old_png_files:\n", + " print \"removing file: \" + old_png_file\n", " os.remove(old_png_file)\n", "\n", " png_filename = './platypus_data_{:s}.png'.format(uuid.uuid4())\n", @@ -272,18 +344,35 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 15, "metadata": { - "collapsed": false, - "deletable": true, - "editable": true + "scrolled": false }, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+EAAABRCAYAAAC9m6cOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAECRJREFUeJzt3XmQZWV5x/HvjxkWQdkkKgIJJKAWCE4QDBZGBQmiErEq\nLhCNoJZRYxRJTAJqXFBTqKkgJoYqF0SJQZDSOMEIEkGtEAdkGxZFIS4sIktAXDDADE/+OG/DnTu9\nzJ3uOd19+X6qpua8513ue8/bb/d9znnPuakqJEmSJEnShrfRfHdAkiRJkqSHC4NwSZIkSZJ6YhAu\nSZIkSVJPDMIlSZIkSeqJQbgkSZIkST0xCJckSZIkqScG4ZIkSZIk9cQgXJIkSZKknhiES5IkSZLU\nk6WjFE52LbhnIjWcO016lLJz3dawh1u/5tBcHp7ZtL1Y2ppN27a1YadMb23VDHXXzM805Yfzpivb\nJWvS7ckM509Xfu284bozlV+/1x21/GzaXij9GLXtuWzr4XC8Rmt7/X/Oh/Pn9T3VQD+mn8pr5w+b\n+vCsnZ6u7kzlZyq7oeoO58/le/R4jfbaG7KtUeqOWnYO666VXZNvT5qephvD+aO+pdkM24Y81Buy\nXzP9ON0C51bVIZNkrWGkILwLwF83RdWNp2l6urxR82eqO5dtzdd7nMu2RjT4yWLJiC87nB6sP5u6\nw/VnU3e4/GzqDqcX6/HakG1Nl57LYz/bfs3m2K+VHviVvHT1UN6a6Y2WrFoze+M185cMlF+ydDhv\nqO5w/kYDdRnKWys91NY05WdTd8O2tf51h+uPXncu25r6fcymH6O3NZufiYXy8zXbn4mF9/M1cj9W\nD6VXDfRj9QNDeUybztCvszW6smqaPIDp6g6nRyk702vZr/nr12zqPhz6NWLd+4fyB6Yy96+eOm/S\nukMvdf8U2zOVnSl/NnWH84fz5rKtuXyPAO+G7SbZvRaXo0uSJEmS1BODcEmSJEmSemIQLkmSJElS\nTwzCJUmSJEnqiUG4JEmSJEk9MQiXJEmSJKknBuGSJEmSJPXEIFySJEmSpJ4YhEuSJEmS1BODcEmS\nJEmSemIQLkmSJElSTwzCJUmSJEnqiUG4JEmSJEk9MQiXJEmSJKknBuGSJEmSJPXEIFySJEmSpJ4Y\nhEuSJEmS1BODcEmSJEmSemIQLkmSJElSTwzCJUmSJEnqiUG4JEmSJEk9MQiXJEmSJKknBuGSJEmS\nJPXEIFySJEmSpJ4YhEuSJEmS1BODcEmSJEmSemIQLkmSJElSTwzCJUmSJEnqiUG4JEmSJEk9MQiX\nJEmSJKknBuGSJEmSJPUkVbXuhZNzgO02XHc0g+2AO+a7E9qgHOPx5viOP8d4/DnG488xHm+O7/ib\nzzG+o6oOmanQSEG45leSS6pqn/nuhzYcx3i8Ob7jzzEef47x+HOMx5vjO/4Wwxi7HF2SJEmSpJ4Y\nhEuSJEmS1BOD8MXlY/PdAW1wjvF4c3zHn2M8/hzj8ecYjzfHd/wt+DH2nnBJkiRJknrilXBJkiRJ\nknpiEL7AJFmS5PIkZ7f0LkkuSnJ9kjOSbNL2b9rS17f8neez31o3SbZOclaSa5N8N8nTk2yb5Lwk\n17X/t2llk+QjbYyvTLL3fPdfM0tyTJJrklyd5PQkmzmPF7ckpyS5LcnVA/tGnrdJjmzlr0ty5Hy8\nF61tivH9UPs9fWWSLybZeiDvuDa+30vy3IH9h7R91yc5tu/3oalNNsYDeX+ZpJJs19LO4UVoqjFO\n8qY2l69J8sGB/c7jRWaK39XLkqxIckWSS5I8re1f8PPYIHzhORr47kD6A8CJVbUrcBfwmrb/NcBd\nbf+JrZwWvpOAc6rqScBT6Mb6WOBrVbUb8LWWBngesFv796fAyf13V6NIsgPwZmCfqnoysAQ4HOfx\nYncqMPydnyPN2yTbAu8Cfg94GvCuicBd8+5U1h7f84AnV9VewPeB4wCS7E43p/dodf453cnzJcBH\n6cZ/d+CIVlYLw6msPcYk2Qk4GLhhYLdzeHE6laExTnIAcBjwlKraA/j7tt95vDidytrz+IPAe6pq\nGfDOloZFMI8NwheQJDsCLwA+0dIBDgTOakU+DbyobR/W0rT857TyWqCSbAU8E/gkQFXdV1U/Y82x\nHB7jz1RnBbB1ku177rZGtxR4RJKlwObALTiPF7Wq+iZw59DuUeftc4HzqurOqrqLLshbKyhQ/yYb\n36r6alWtaskVwI5t+zDgc1V1b1X9ELie7oPc04Drq+oHVXUf8LlWVgvAFHMYupOffw0MPiDJObwI\nTTHGbwBOqKp7W5nb2n7n8SI0xRgXsGXb3gr4Sdte8PPYIHxh+TDdH4MHWvrRwM8GPgjcBOzQtncA\nbgRo+Xe38lq4dgFuBz6V7paDTyTZAnhsVd3SyvwUeGzbfnCMm8Hx1wJUVTfTnWm/gS74vhu4FOfx\nOBp13jqfF69XA19p247vmEhyGHBzVa0cynKMx8cTgN9vt3t9I8m+bb9jPD7eAnwoyY10n7+Oa/sX\n/BgbhC8QSQ4FbquqS+e7L9pglgJ7AydX1e8Cv+KhJawAVPd1BX5lwSLVljQdRnfC5fHAFnilZOw5\nb8dXkrcDq4DPzndfNHeSbA68jW75qsbXUmBbYD/gr4AzXW02dt4AHFNVOwHH0FabLgYG4QvH/sAL\nk/yIbvnLgXT3D2/dlrVCtxzu5rZ9M7ATQMvfCvjfPjuskd0E3FRVF7X0WXRB+a0Ty8zb/xPLpR4c\n42Zw/LUwHQT8sKpur6r7gS/QzW3n8fgZdd46nxeZJEcBhwIvr4e+z9XxHQ+/Q3eydGX73LUjcFmS\nx+EYj5ObgC+0JckX06003Q7HeJwcSfdZC+DzdLcUwCIYY4PwBaKqjquqHatqZ7qHRZxfVS8HLgBe\n3IodCXypbS9vaVr++QMfErQAVdVPgRuTPLHteg7wHdYcy+ExfmV7wuN+wN0Dy1+1MN0A7Jdk83a2\nfWKMncfjZ9R5ey5wcJJt2oqJg9s+LUBJDqG7PeyFVXXPQNZy4PB032ywC91Dfy4Gvg3slu6bEDah\n+zu+vO9+a91U1VVV9Ziq2rl97roJ2Lv9nXYOj49/Aw4ASPIEYBPgDpzH4+QnwLPa9oHAdW17wc/j\npTMX0Tz7G+BzSd4HXM5Dyyw+CZyW5Hq6hxQcPk/902jeBHy2/XL/AfAqupNhZyZ5DfBj4KWt7H8A\nz6d7YMg9rawWsKq6KMlZwGV0S1gvBz4GfBnn8aKV5HTg2cB2SW6ie7LqCYwwb6vqziTvpfuQB3B8\nVU32oCj1bIrxPQ7YFDivrV5dUVWvr6prkpxJd3JtFfDGqlrd2vlzug9zS4BTquqa3t+MJjXZGFfV\nVMtWncOL0BTz+BTglHRfaXUfcGQ70e08XoSmGOPXAie11YT/R/ckdFgE8zhedJEkSZIkqR8uR5ck\nSZIkqScG4ZIkSZIk9cQgXJIkSZKknhiES5IkSZLUE4NwSZIkSZJ6YhAuSVowkqxOcsXAv2Pb/q8n\n2Wc92luW5PnT5O+T5CPr2ddJ+5Rk4yQnJLkuyWVJvpXkeevzGpO0/cv1rLfGcUjywoljO02d45Mc\n1LbfkmTzEV8zSc5PsuUM5fZLclEb7+8mefdA3sZJLhvldad4jf9s3wkrSdK883vCJUkLya+ratkc\ntrcM2IfuO0PXkGRpVV0CXDKHrwfwXmB74MlVdW+SxwLPmuPXGNUax6GqlgPLp6tQVe8cSL4F+Be6\n71tdV88HVlbVz2co92ngpVW1MskS4IkDec8ALhzhNadyGvBnwPvnoC1JkmbFK+GSpEUlycHt6vJl\nST6f5JFt/75J/jvJyiQXJ9kKOB54WbvK+rIk705yWpILgdOSPDvJ2a3+I5N8KslVSa5M8kdt/8lJ\nLklyTZL3zNC3zYHXAm+qqnsBqurWqjqz5R/R2r86yQcG6v0yyftb31e0wJ0ku7T3elWS9w2Uf7Df\nLf1PSY4a4Tgc1epsleTHSTZqdbdIcmO7An1qkhcneTPweOCCJBckeXWSDw+89muTnDjJ4Xg58KVW\nZuckVw/UeevAFe/HALe0Y7W6qr4z0MYhwFdanVe2cVmZ5LS279Q2PiuS/KAdl1PaFfVTB9pZDhwx\n3dhJktQXg3BJ0kLyiKy5HP1lg5lJtgPeARxUVXvTXcX+iySbAGcAR1fVU4CDgF8B7wTOqKplVXVG\na2b3Vn84KPtb4O6q2rOq9gLOb/vfXlX7AHsBz0qy1zT93xW4YbKrv0keD3wAOJDuyvS+SV7UsrcA\nVrS+f5MukAc4CTi5qvakBarTGfE4UFV3A1fw0JX6Q4Fzq+r+gTIfAX4CHFBVBwBnAn+YZONW5FXA\nKZN0Z3/g0pn6DJwIfC/JF5O8LslmA3kHAF9PsgfduB/Y3tfRA2W2AZ4OHEMXbJ8I7AHsmWRZew93\nAZsmefQ69EeSpA3KIFyStJD8ugWKy4YDxmY/uiD6wiRXAEcCv0W3hPmWqvo2QFX9vKpWTfEay6vq\n15PsPwj46ESiBW4AL233JV9OF9ztvp7vbV/g61V1e+vbZ4Fntrz7gIkr25cCO7ft/YHT2/Zp6/Aa\noxyHCWcAEyc7Dm/pKVXVL+lOUBya5EnAxlV11SRFt62qX8zU4ao6nm6p/FeBPwbOAUiyA3BnVd1D\nd+Li81V1R6tz50AT/15VBVwF3FpVV1XVA8A1PHQcAW6ju6IvSdK88p5wSdJiEuC84avYSfYcoY1f\nrfOLJbsAbwX2raq72hLnzaapcj3wm0m2XId7oQfd3wJJgNWs+fe5Jim/ijVPpE/Xp5ksB/4uybbA\nU3loBcB0PgG8DbgW+NQUZVYl2agFxNP2t6r+Bzg5yceB29sV60OAc9ehL/e2/x8Y2J5IDx7HzYDJ\nTr5IktQrr4RLkhaTFcD+SXaFB+9hfgLwPWD7JPu2/Y9KshT4BfCodWz7POCNE4n2NO0t6YL2u9t9\n2tM+5bxdtf0kcFJbGk6S30jyEuBiuuXs27UHkB0BfGOGPl1Id3UaunusJ/wY2D3Jpkm2Bp7T9o98\nHNqV7W/TLX0/u6pWT1JsjfpVdRGwE92V69MnKT/Rl99u27cCj0ny6CSb0i17p/XxBUnSkrvRnYT4\nGQP3g9OdGHjJxHLydsJgnbX2Hwf8aJR6kiRtCAbhkqSFZPie8BMGM6vqduAo4PQkVwLfAp5UVffR\nLan+xyQr6QLqzYAL6ILVte4vn8T7gG3aQ9NW0t0DvZJuGfq1wL+ybk/qfgdwO/Cd9jCys4GfV9Ut\nwLGtTyuBS6vqSzO0dTTwxiRXATsMHIcb6e7Nvrr9f3nbv77H4QzgFUy9FP1jwDlJLhjYdyZw4cCy\n/WFfBp7d+nU/3cPhLm59unag3J/Q3RN+Bd2S+4mTDbtW1bWt/jV0Tzb/Rntf/zDFa07lqXT33M+0\nNF+SpA0uD61+kyRJWjft6ewnVtXXpsjfHvhMVf3BerT9DOAVVfX6WXZzor2T6J4FMGlfJUnqk1fC\nJUnSOkuydZLv0z1Eb8qgtl35/3iSLUd9jar6r7kKwJurDcAlSQuFV8IlSZIkSeqJV8IlSZIkSeqJ\nQbgkSZIkST0xCJckSZIkqScG4ZIkSZIk9cQgXJIkSZKknhiES5IkSZLUk/8Ht8qyghMh6V0AAAAA\nSUVORK5CYII=\n", + "application/vnd.jupyter.widget-view+json": { + "model_id": "faf94d69a0094da2b0179e36f986344b", + "version_major": 2, + "version_minor": 0 + }, + "text/html": [ + "

Failed to display Jupyter Widget of type Map.

\n", + "

\n", + " If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n", + " that the widgets JavaScript is still loading. If this message persists, it\n", + " likely means that the widgets JavaScript library is either not installed or\n", + " not enabled. See the Jupyter\n", + " Widgets Documentation for setup instructions.\n", + "

\n", + "

\n", + " If you're reading this message in another frontend (for example, a static\n", + " rendering on GitHub or NBViewer),\n", + " it may mean that your frontend doesn't currently support widgets.\n", + "

\n" + ], "text/plain": [ - "" + "Map(center=[51.983079299809056, 4.288419291357442], layers=(TileLayer(attribution=u'Map data (c) OpenStreetMap contributors', options=[u'opacity', u'attribution', u'max_zoom', u'detect_retina', u'min_zoom', u'tile_size'], url=u'https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png'), ImageOverlay(bounds=[[51.98242187260349, 4.285285712486869], [51.98466966749721, 4.28902748295083]], options=[u'opacity', u'attribution'], url=u'./platypus_data_c40dd200-a93e-4ba2-9a7e-b3c193762c00.png')), layout=Layout(align_self=u'stretch', height=u'400px'), options=[u'keyboard_pan_offset', u'tap', u'attribution_control', u'max_zoom', u'min_zoom', u'bounce_at_zoom_limits', u'keyboard', u'scroll_wheel_zoom', u'dragging', u'inertia_max_speed', u'close_popup_on_click', u'zoom_control', u'box_zoom', u'double_click_zoom', u'tap_tolerance', u'zoom_start', u'keyboard_zoom_offset', u'inertia_deceleration', u'inertia', u'center', u'zoom', u'world_copy_jump', u'zoom_animation_threshold', u'touch_zoom'], zoom=14)" ] }, "metadata": {}, @@ -291,9 +380,10 @@ }, { "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c6b7440ca5544c89bdf73d6c78e00d70" - } + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+EAAABRCAYAAAC9m6cOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAADbVJREFUeJzt3X+sJWV9x/H3h92VFZSfWymgFgxU\ng4IrQovBKAq1qFRNqqKVij9qbWoq2pjGWuuvaqNJ4wq1IfEHoNuWgqStW9pCqYCmVEARVkCwUKsi\nXWEpuKgQ3V2+/WOeC+eevffcPfeeO3d3eb+SzZ1nnh/zzNwzd/Y7zzNzUlVIkiRJkqTFt9tSd0CS\nJEmSpEcLg3BJkiRJknpiEC5JkiRJUk8MwiVJkiRJ6olBuCRJkiRJPTEIlyRJkiSpJwbhkiRJkiT1\nxCBckiRJkqSeGIRLkiRJktST5eMUTg4reGAqNZw7Ij1O2Um3NezR1q8JmuThWUjbO0tbC2nbthb3\nlOmtrZqj7vT8jCg/nDeqbJesGZdnMpw/qvy2ecN15yo/v+2OW34hbe8o/Ri37Um29Wg4XuO1Pf/P\n+XD+ku5TDfRj9Km8bf6w2Q/PtulRdecqP1fZxao7nD/JffR4jbftxWxrnLrjlp1g3W2ya+blGdMj\nujGcP+4uLeTXtpiHejH7NdfHaQNcWlUnz5A1zVhBeBeAv3WWqitGND0qb9z8uepOsq2l2sdJtjWm\nwf9ZLBtzs8PpwfoLqTtcfyF1h8svpO5wemc9XovZ1qj0JI/9Qvu1kGO/TXrgT/LyrUN509O7Ldsy\nPXvF9PxlA+WXLR/OG6o7nL/bQF2G8rZJD7U1ovxC6i5uW/OvO1x//LqTbGv2/VhIP8ZvayGfiR3l\n87XQz8SO9/kaux9bh9JbBvqx9aGhPEamM/TnbFpXtozIAxhVdzg9Ttm5tmW/lq5fC6n7aOjXmHU3\nD+UPnMps3jp73ox1hza1eZblucrOlb+QusP5w3mTbGuS+wjwAVg1w+ptOB1dkiRJkqSeGIRLkiRJ\nktQTg3BJkiRJknpiEC5JkiRJUk8MwiVJkiRJ6olBuCRJkiRJPTEIlyRJkiSpJwbhkiRJkiT1xCBc\nkiRJkqSeGIRLkiRJktQTg3BJkiRJknpiEC5JkiRJUk8MwiVJkiRJ6olBuCRJkiRJPTEIlyRJkiSp\nJwbhkiRJkiT1xCBckiRJkqSeGIRLkiRJktQTg3BJkiRJknpiEC5JkiRJUk8MwiVJkiRJ6olBuCRJ\nkiRJPTEIlyRJkiSpJwbhkiRJkiT1xCBckiRJkqSeGIRLkiRJktQTg3BJkiRJknpiEC5JkiRJUk8M\nwiVJkiRJ6olBuCRJkiRJPTEIlyRJkiSpJ6mq7S+cXAKsWrzuSLu8VcA9S90JaRfl+SUtHs8vaXF4\nbu1a7qmqk+cqNFYQLmlhkny9qo5Z6n5IuyLPL2nxeH5Ji8Nz69HJ6eiSJEmSJPXEIFySJEmSpJ4Y\nhEv9+tRSd0DahXl+SYvH80taHJ5bj0I+Ey5JkiRJUk8cCZckSZIkqScG4dKEJVmW5PokF7f0oUmu\nSXJ7kguSPKat372lb2/5hyxlv6UdXZJ9klyU5NYktyR5TpL9klyW5Lb2c99WNknOaufXN5McvdT9\nl3ZkSd6Z5OYkNyU5P8lKr1/S/CQ5J8ndSW4aWDf29SrJ6a38bUlOX4p90eIwCJcm7wzgloH0x4A1\nVXUYcB/w5rb+zcB9bf2aVk7S7M4ELqmqpwHPpDvP3g18qaoOB77U0gAvBg5v/34XOLv/7ko7hyQH\nA28HjqmqZwDLgNfg9Uuar/OA4e+KHut6lWQ/4P3ArwK/Arx/KnDXzs8gXJqgJE8EXgp8pqUDvBC4\nqBX5HPCKtvzylqbln9jKSxqSZG/gecBnAarq51X1I6afR8Pn1+erczWwT5IDe+62tDNZDjw2yXJg\nD2ADXr+keamqrwD3Dq0e93r168BlVXVvVd0HXMa2gb12Ugbh0mR9Avgj4KGW3h/4UVVtaekfAAe3\n5YOBOwBa/qZWXtK2DgU2Aue2xz0+k2RP4ICq2tDK/BA4oC0/fH41g+eepAFVdSfwF8D36YLvTcB1\neP2SJmnc65XXsV2YQbg0IUlOAe6uquuWui/SLmg5cDRwdlU9C/gpj0zlA6C6r/vwKz+kMbUpri+n\nu9l1ELAnjrhJi8brlQzCpck5HnhZku8Cf0c3je9MumlFy1uZJwJ3tuU7gScBtPy9gf/rs8PSTuQH\nwA+q6pqWvoguKL9rapp5+3l3y3/4/GoGzz1J050E/E9VbayqzcDf013TvH5JkzPu9crr2C7MIFya\nkKr646p6YlUdQvdCm8ur6nXAFcArW7HTgS+25XUtTcu/vN0ZlTSkqn4I3JHkqW3VicC3mH4eDZ9f\nr29vnT0O2DQwDVDSdN8HjkuyR3u2e+r88volTc6416tLgRcl2bfNVnlRW6ddQPybKU1ekhOAd1XV\nKUmeQjcyvh9wPXBaVf0syUpgLfAsupd3vKaqvrNUfZZ2dElW07308DHAd4A30t1MvhB4MvA94NVV\ndW8LJD5JN6X2AeCNVfX1Jem4tBNI8kHgVGAL3bXqd+ieP/X6JY0pyfnACcAq4C66t5z/I2Ner5K8\nCXhPa/YjVXVun/uhxWMQLkmSJElST5yOLkmSJElSTwzCJUmSJEnqiUG4JEmSJEk9MQiXJEmSJKkn\nBuGSJEmSJPXEIFyStMNIsjXJDQP/3t3WX5nkmHm0tzrJS0bkH5PkrHn2dcY+JVmR5KNJbkvyjSRf\nTfLi+WxjhrZ/Ms96045DkpdNHdsRdT6U5KS2/I4ke4y5zSS5PMlec5Q7Lsk17fd9S5IPDOStSPKN\ncbY7yzb+vX3PriRJS275UndAkqQBD1bV6gm2txo4BviX4Ywky9t3sU76+8P/DDgQeEb7TuUDgOdP\neBvjmnYcqmodsG5Uhap630DyHcBf032H7fZ6CbC+qu6fo9zn6L4vd32SZcBTB/KeC1w1xjZnsxb4\nfeAjE2hLkqQFcSRckrRTSfKiNrr8jSRfSPK4tv7YJP+ZZH2Sa5PsDXwIOLWNsp6a5ANJ1ia5Clib\n5IQkF7f6j0tybpIbk3wzyW+29Wcn+XqSm5N8cI6+7QG8BfiDqvoZQFXdVVUXtvzXtvZvSvKxgXo/\nSfKR1verW+BOkkPbvt6Y5MMD5R/ud0t/MskbxjgOb2h19k7yvSS7tbp7JrmjjUCfl+SVSd4OHARc\nkeSKJG9K8omBbb8lyZoZDsfrgC+2MockuWmgzrsGRryfAGxox2prVX1roI2TgX9tdV7ffi/rk6xt\n685rv5+rk3ynHZdz2oj6eQPtrANeO+p3J0lSXwzCJUk7ksdm+nT0Uwczk6wC3gucVFVH041i/2GS\nxwAXAGdU1TOBk4CfAu8DLqiq1VV1QWvmiFZ/OCj7U2BTVR1ZVUcBl7f1f1JVxwBHAc9PctSI/h8G\nfH+m0d8kBwEfA15INzJ9bJJXtOw9gatb379CF8gDnAmcXVVH0gLVUcY8DlTVJuAGHhmpPwW4tKo2\nD5Q5C/hf4AVV9QLgQuA3kqxoRd4InDNDd44Hrpurz8Aa4NtJ/iHJW5OsHMh7AXBlkqfT/d5f2Pbr\njIEy+wLPAd5JF2yvAZ4OHJlkdduH+4Ddk+y/Hf2RJGlRGYRLknYkD7ZAcfVwwNgcRxdEX5XkBuB0\n4JfopjBvqKqvAVTV/VW1ZZZtrKuqB2dYfxLwV1OJFrgBvLo9l3w9XXB3xDz37Vjgyqra2Pr2N8Dz\nWt7PgamR7euAQ9ry8cD5bXntdmxjnOMw5QJg6mbHa1p6VlX1E7obFKckeRqwoqpunKHoflX147k6\nXFUfopsq/2/AbwGXACQ5GLi3qh6gu3Hxhaq6p9W5d6CJf6qqAm4E7qqqG6vqIeBmHjmOAHfTjehL\nkrSkfCZckrQzCXDZ8Ch2kiPHaOOn272x5FDgXcCxVXVfm+K8ckSV24EnJ9lrO56FHrS5BZIAW5l+\nfa4Zym9h+o30UX2ayzrgz5PsBzybR2YAjPIZ4D3ArcC5s5TZkmS3FhCP7G9V/TdwdpJPAxvbiPXJ\nwKXb0ZeftZ8PDSxPpQeP40pgppsvkiT1ypFwSdLO5Grg+CSHwcPPMP8y8G3gwCTHtvWPT7Ic+DHw\n+O1s+zLgbVOJ9jbtveiC9k3tOe2Rbzlvo7afBc5sU8NJ8gtJXgVcSzedfVV7AdlrgS/P0aer6Ean\noXvGesr3gCOS7J5kH+DEtn7s49BGtr9GN/X94qraOkOxafWr6hrgSXQj1+fPUH6qL09py3cBT0iy\nf5Ld6aa90/r40iRpycPpbkL8iIHnweluDLxqajp5u2Gw3Vr7vwh8d5x6kiQtBoNwSdKOZPiZ8I8O\nZlbVRuANwPlJvgl8FXhaVf2cbkr1XyZZTxdQrwSuoAtWt3m+fAYfBvZtL01bT/cM9Hq6aei3An/L\n9r2p+73ARuBb7WVkFwP3V9UG4N2tT+uB66rqi3O0dQbwtiQ3AgcPHIc76J7Nvqn9vL6tn+9xuAA4\njdmnon8KuCTJFQPrLgSuGpi2P+yfgRNavzbTvRzu2tanWwfK/TbdM+E30E25n7rZcFhV3drq30z3\nZvMvt/36+CzbnM2z6Z65n2tqviRJiy6PzH6TJEnaPu3t7Guq6kuz5B8IfL6qfm0ebT8XOK2qfm+B\n3Zxq70y6dwHM2FdJkvrkSLgkSdpuSfZJ8l90L9GbNahtI/+fTrLXuNuoqv+YVADe3GQALknaUTgS\nLkmSJElSTxwJlyRJkiSpJwbhkiRJkiT1xCBckiRJkqSeGIRLkiRJktQTg3BJkiRJknpiEC5JkiRJ\nUk/+H/DgIpMMtIAdAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] }, "metadata": {}, "output_type": "display_data" @@ -304,8 +394,8 @@ "# Create a map centered on this data log.\n", "center = [pose['latitude'].median(), pose['longitude'].median()]\n", "# print center\n", - "zoom = 17\n", - "m = Map(center=center, zoom=zoom, height='1000px')\n", + "zoom = 14\n", + "m = Map(center=center, zoom=zoom, height='1300px')\n", "if sensor_name in data:\n", " m += io # Add image overlay\n", "if sensor_name not in data:\n", @@ -325,18 +415,16 @@ " orientation='horizontal')\n", " cb1.set_label(sensor_units)\n", "\n", - "pyplot.show()\n", + "png_filename = './platypus_data_{:s}.png'.format(uuid.uuid4())\n", + "pyplot.savefig(png_filename)\n", + "# pyplot.show()\n", "m" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true, - "deletable": true, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [] } @@ -373,5 +461,5 @@ } }, "nbformat": 4, - "nbformat_minor": 0 + "nbformat_minor": 1 } From f7f63a825c2a998f8aa31d09f7b61e1b5bfc22ca Mon Sep 17 00:00:00 2001 From: Shawn Hanna Date: Thu, 7 Dec 2017 09:40:23 +0000 Subject: [PATCH 24/43] updated histogram plotting example --- examples/histogram.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/examples/histogram.py b/examples/histogram.py index ee271e8..28cec48 100644 --- a/examples/histogram.py +++ b/examples/histogram.py @@ -56,14 +56,23 @@ def trim_using_EC(dataframe, threshold=100): return dataframe -folders = ['/home/shawn/day4','/home/shawn/day3','/home/shawn/day2','/home/shawn/day1'] +folders = ['/home/shawn/day4','/home/shawn/day3','/home/shawn/day2','/home/shawn/day1', '/home/shawn/all'] for folder in folders: data = platypus.io.logs.merge_files(glob.glob(folder+'/*.txt')) data = trim_using_EC(data, 200) - channel = 'ph' - sensor ='ATLAS_PH' + # channel = 'ph' + # sensor ='ATLAS_PH' + # bins = np.arange(5.5, 9 + 0.5, 0.25) + + # channel = 'ec' + # sensor ='ES2' + # bins = np.arange(200,1800,100) + + channel = 'temperature' + sensor ='ES2' + bins = np.arange(6,13,0.5) # Get the standard deviation of the ES2 data. es2_stddev = data[sensor][channel].std() @@ -73,16 +82,15 @@ def trim_using_EC(dataframe, threshold=100): es2_mean = data[sensor][channel].mean() print channel+" mean", es2_mean - # bins = np.arange(6,13,0.5) - # bins = np.arange(200,1800,100) - bins = np.arange(5.5, 9 + 0.5, 0.25) # n, bins, patches = plt.hist(data[sensor][channel], bins=xrange(200,1600,100)) - n, bins, patches = plt.hist(data[sensor][channel], normed=False, bins=bins) + weights = np.ones_like(data[sensor][channel])/float(len(data[sensor][channel])) + n, bins, patches = plt.hist(data[sensor][channel], weights=weights, bins=bins) print n, bins, patches plt.xlabel(channel) - plt.ylabel('Counts') + plt.ylabel('Percentage') + plt.ylim(0,1) plt.title('Histogram of ' + channel + ' ' + folder.split('/')[-1]) plt.savefig('Histogram of ' + channel + ' ' + folder.split('/')[-1]+'.png') # plt.text(0, .25, "Standard Dev: " + str(es2_stddev)) From 8b0756718855f33ac37081646b5bd85301cdae56 Mon Sep 17 00:00:00 2001 From: Shawn Hanna Date: Fri, 6 Apr 2018 01:34:14 -0400 Subject: [PATCH 25/43] more updates to histogram and jupyter notebook. added dependencies to setup.py --- examples/histogram.py | 108 ++++++++++---- examples/histogram_gui.py | 26 ++++ examples/loading_logs.py | 63 ++++---- notebooks/Data_Interpolation.ipynb | 226 +++++++++++++++++------------ setup.py | 7 +- 5 files changed, 275 insertions(+), 155 deletions(-) create mode 100644 examples/histogram_gui.py diff --git a/examples/histogram.py b/examples/histogram.py index 28cec48..2991e49 100644 --- a/examples/histogram.py +++ b/examples/histogram.py @@ -18,7 +18,7 @@ import glob import os import numpy as np -import plotly.plotly as py +import math # Read the data log from file. # Note: for log versions <5.0, this filename must be 'airboat_[timestamp].txt]. @@ -56,45 +56,93 @@ def trim_using_EC(dataframe, threshold=100): return dataframe -folders = ['/home/shawn/day4','/home/shawn/day3','/home/shawn/day2','/home/shawn/day1', '/home/shawn/all'] +def load_data(folders = [], files = [], ec_trim_value = 50): + if (len(folders) == 0): + folders = [sys.path.cwd] -for folder in folders: - data = platypus.io.logs.merge_files(glob.glob(folder+'/*.txt')) - data = trim_using_EC(data, 200) + for folder in folders: + files.extend(glob.glob(folder+'/*.txt')) - # channel = 'ph' - # sensor ='ATLAS_PH' - # bins = np.arange(5.5, 9 + 0.5, 0.25) + print files + # todo: remove duplicates? - # channel = 'ec' - # sensor ='ES2' - # bins = np.arange(200,1800,100) - - channel = 'temperature' - sensor ='ES2' - bins = np.arange(6,13,0.5) + data = platypus.io.logs.merge_files(files) + data = trim_using_EC(data, ec_trim_value) + return data + +def plot_hist_sensor(data, sensor = 'ES2', channel='ec', num_bins = 10, hide_top_n_percent = 0, hide_bot_n_percent = 0, save_dir = "~/save"): + num_readings = len(data[sensor][channel]) + # Get the std of the data + sensor_stddev = data[sensor][channel].std() + # Get the mean of the data + sensor_mean = data[sensor][channel].mean() + # Get the min of the data + sensor_min = data[sensor][channel].min() + # Get the max of the data + sensor_max = data[sensor][channel].max() + + print channel+" number of readings", num_readings + print channel+" std", sensor_stddev + print channel+" mean", sensor_mean + print channel+" min", sensor_min + print channel+" max", sensor_max - # Get the standard deviation of the ES2 data. - es2_stddev = data[sensor][channel].std() - print channel +" std", es2_stddev + hist_max = math.ceil(sensor_max - hide_top_n_percent * 0.01 * sensor_max) + hist_min = math.floor(sensor_min + hide_bot_n_percent * 0.01 * sensor_min) + bin_size = (hist_max - hist_min)/float(num_bins) - # Get the mean of the ES2 data. - es2_mean = data[sensor][channel].mean() - print channel+" mean", es2_mean + bins = np.arange(hist_min, hist_max, bin_size) + # print hist_max, hist_min, bin_size, bins # n, bins, patches = plt.hist(data[sensor][channel], bins=xrange(200,1600,100)) - weights = np.ones_like(data[sensor][channel])/float(len(data[sensor][channel])) - n, bins, patches = plt.hist(data[sensor][channel], weights=weights, bins=bins) + weights = np.ones_like(data[sensor][channel])/float(num_readings) * 100 + if (num_bins <= 0): + n, bins, patches = plt.hist(data[sensor][channel], weights=weights) + else: + n, bins, patches = plt.hist(data[sensor][channel], weights=weights, bins=bins) - print n, bins, patches + # print n, bins, patches plt.xlabel(channel) - plt.ylabel('Percentage') - plt.ylim(0,1) - plt.title('Histogram of ' + channel + ' ' + folder.split('/')[-1]) - plt.savefig('Histogram of ' + channel + ' ' + folder.split('/')[-1]+'.png') + plt.ylabel('Percentage of values in the given range') + plt.ylim(0,50) + plt.title('Histogram of ' + channel + ' ' + save_dir.split('/')[-1]) + plt.savefig(save_dir + "/"+'Histogram of ' + channel + ' ' + save_dir.split('/')[-1]+'.png') # plt.text(0, .25, "Standard Dev: " + str(es2_stddev)) - plt.figtext(.16, .75, "Mean: " + str(es2_mean)) - plt.figtext(.16, .7, "std: " + str(es2_stddev)) + plt.figtext(.16, .75, "Mean: " + str(sensor_mean)) + plt.figtext(.16, .7, "std: " + str(sensor_stddev)) plt.grid(True) plt.show() + +def get_folders(): + # folders = ['/home/shawn/NL2/grokeneveldse_polder/grokeneveldse_polder_feb_2018/'] + folders = ['/home/shawn/NL1/all_nov_2017'] + return folders + +def main(): + print "enter EC trim value: " + new_ec_trim = int(raw_input()) + + folders = get_folders() + + data = load_data(folders=folders, ec_trim_value = new_ec_trim) + while ( True ): + print "what would you like to do?\n0: quit\n1: plot EC\n2: plot pH\n3: plot DO\n4: plot temp\n5: change EC trim value" + command = raw_input() + + if (command == '0'): + break + elif (command == '1'): + plot_hist_sensor(data, 'ES2', 'ec', num_bins = 10, hide_top_n_percent = 0, save_dir = folders[0]) + elif (command == '2'): + plot_hist_sensor(data, 'ATLAS_PH', 'ph', num_bins = 10, hide_bot_n_percent = 0, save_dir = folders[0]) + elif (command == '3'): + plot_hist_sensor(data, 'ATLAS_DO', 'do', num_bins = 10, save_dir = folders[0]) + elif (command == '4'): + plot_hist_sensor(data, 'ES2', 'temperature', num_bins = 10, save_dir = folders[0]) + else: + print command +" is not valid" + + +if __name__ == '__main__': + main() diff --git a/examples/histogram_gui.py b/examples/histogram_gui.py new file mode 100644 index 0000000..c835115 --- /dev/null +++ b/examples/histogram_gui.py @@ -0,0 +1,26 @@ +from Tkinter import Tk, BOTH + +class Example(Frame): + + def __init__(self): + super().__init__() + + self.initUI() + + + def initUI(self): + + self.master.title("Simple") + self.pack(fill=BOTH, expand=1) + + +def main(): + + root = Tk() + root.geometry("250x150+300+300") + app = Example() + root.mainloop() + + +if __name__ == '__main__': + main() diff --git a/examples/loading_logs.py b/examples/loading_logs.py index e91ef84..64d861b 100644 --- a/examples/loading_logs.py +++ b/examples/loading_logs.py @@ -56,46 +56,47 @@ def trim_using_EC(dataframe, threshold=100): sensor = "ES2" -channel = 'temperature' +channel = 'ec' -for folder in glob.glob('/home/shawn/day*/'): - print "processing folder: ", folder - for file in glob.glob(folder+'*.txt'): - print file - out_name = os.path.basename(os.path.splitext(file)[0]) - print out_name +for folder in glob.glob('/home/shawn/NL2/nordpolder_van_delfgauw/'): + print "processing folder: ", folder + for file in glob.glob(folder+'platypus_20180215_104248.txt'): + if (os.path.exists(file)): + print file + out_name = os.path.basename(os.path.splitext(file)[0]) + print out_name - data = platypus.io.logs.merge_files(glob.glob(folder+"/*.txt")) - data = trim_using_EC(data, 200) + data = platypus.io.logs.merge_files(glob.glob(folder+"/*.txt")) + data = trim_using_EC(data, 100) - # Access the first 100 GPS locations for the vehicle. - poses = data['pose'] + # Access the first 100 GPS locations for the vehicle. + poses = data['pose'] - output_base = folder+out_name+"_"+channel + output_base = folder+out_name+"_"+channel - plt.title("Path of vehicle: " + out_name) - # Plot the first 100 GPS locations as UTM coordinates using matplotlib. - plt.plot(poses['easting'], poses['northing']) - plt.savefig(output_base + "_path.png") - # plt.show() - plt.clear() + # plt.title("Path of vehicle: " + out_name) + # # Plot the first 100 GPS locations as UTM coordinates using matplotlib. + # plt.plot(poses['easting'], poses['northing']) + # plt.savefig(output_base + "_path.png") + # plt.show() + # # plt.clear() - # Retrieve temperature data from the ES2 sensor. - # temp = data[sensor]['temp'] + # Retrieve temperature data from the ES2 sensor. + # temp = data[sensor]['temp'] - # Plot ES2 electrical conductivity data using matplotlib. + # Plot ES2 electrical conductivity data using matplotlib. - # print data[sensor][channel] + # print data[sensor][channel] - plt.title("Graph of "+channel+" data: " + out_name) - plt.plot(data[sensor].index, data[sensor][channel]) - # plt.savefig(folder+"_"+channel+"_graph.png") - plt.savefig(output_base + "_graph.png") + plt.title("Graph of "+channel+" data: " + out_name) + plt.plot(data[sensor].index, data[sensor][channel]) + # plt.savefig(folder+"_"+channel+"_graph.png") + plt.savefig(output_base + "_graph.png") - # plt.show() - plt.clear() + plt.show() + # plt.clear() - # Get the standard deviation of the ES2 data. - es2_stddev = data[sensor].std() - print "deviation", es2_stddev + # Get the standard deviation of the ES2 data. + es2_stddev = data[sensor].std() + print "deviation", es2_stddev diff --git a/notebooks/Data_Interpolation.ipynb b/notebooks/Data_Interpolation.ipynb index 6c53beb..6f370a1 100644 --- a/notebooks/Data_Interpolation.ipynb +++ b/notebooks/Data_Interpolation.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -25,7 +25,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 14, "metadata": { "scrolled": false }, @@ -47,19 +47,12 @@ ], "source": [ "# Import the data from the specified logfile\n", - "# log_path = \"/home/shawn/day2/\"\n", - "# log_filenames = [\n", - "# log_path + \"platypus_20171119_081603.txt\",\n", - "# log_path + \"platypus_20171119_093819.txt\",\n", - "# log_path + \"platypus_20171119_102933.txt\",\n", - "# log_path + \"platypus_20171119_105652.txt\",\n", - "# log_path + \"platypus_20171119_105709.txt\",\n", - "# log_path + \"platypus_20171119_112232.txt\"\n", - "# ]\n", "\n", - "log_path = \"/home/shawn/day1/\"\n", + "log_ext = \".txt\"\n", + "\n", + "log_path = \"/home/shawn/NL2/day1/\"\n", "log_filenames = [\n", - " log_path + \"platypus_20171117_034941.txt\",\n", + "# log_path + \"platypus_20171117_034941.txt\",\n", "# log_path + \"platypus_20171117_045142.txt\",\n", "# log_path + \"platypus_20171117_052611.txt\",\n", "# log_path + \"platypus_20171117_063759.txt\",\n", @@ -72,38 +65,77 @@ "# log_path + \"platypus_20171117_103741.txt\",\n", " ]\n", "\n", - "log_path = \"/home/shawn/day4/\"\n", - "log_filenames = [\n", - " log_path + \"platypus_20171121_045529.txt\",\n", - "# log_path + \"platypus_20171121_053515.txt\",\n", - "# log_path + \"platypus_20171121_073237.txt\",\n", + "# log_path = \"/home/shawn/day 2/\"\n", + "# log_filenames = [\n", + "# log_path + \"platypus_20171119_081603.txt\",\n", + "# # log_path + \"platypus_20171119_093819.txt\",\n", + "# # log_path + \"platypus_20171119_102933.txt\",\n", + "# # log_path + \"platypus_20171119_105652.txt\",\n", + "# # log_path + \"platypus_20171119_105709.txt\",\n", + "# # log_path + \"platypus_20171119_112232.txt\"\n", + "# ]\n", + "\n", + "# log_path = \"/home/shawn/day 3/\"\n", + "# log_filenames = [\n", + "# # log_path + \"platypus_20171120_040729.txt\",\n", + "# # log_path + \"platypus_20171120_052156.txt\",\n", + "# # log_path + \"platypus_20171120_065410.txt\",\n", + "# log_path + \"platypus_20171120_072029.txt\",\n", + "# ]\n", + "\n", + "# log_path = \"/home/shawn/day 4/\"\n", + "# log_filenames = [\n", + "# # log_path + \"platypus_20171121_045529.txt\",\n", + "# # log_path + \"platypus_20171121_053515.txt\",\n", + "# # log_path + \"platypus_20171121_073237.txt\",\n", "# log_path + \"platypus_20171121_074700.txt\",\n", - "# log_path + \"platypus_20171121_084718.txt\",\n", - "# log_path + \"platypus_20171121_092302.txt\",\n", - "# log_path + \"platypus_20171121_095421.txt\",\n", - "# log_path + \"platypus_20171121_101351.txt\",\n", - "# log_path + \"platypus_20171121_103427.txt\",\n", - "# log_path + \"platypus_20171121_103725.txt\",\n", - "# log_path + \"platypus_20171121_110202.txt\",\n", - "# log_path + \"platypus_20171121_113356.txt\"\n", - " ]\n", + "# # log_path + \"platypus_20171121_084718.txt\",\n", + "# # log_path + \"platypus_20171121_092302.txt\",\n", + "# # log_path + \"platypus_20171121_095421.txt\",\n", + "# # log_path + \"platypus_20171121_101351.txt\",\n", + "# # log_path + \"platypus_20171121_103427.txt\",\n", + "# # log_path + \"platypus_20171121_103725.txt\",\n", + "# # log_path + \"platypus_20171121_110202.txt\",\n", + "# # log_path + \"platypus_20171121_113356.txt\"\n", + "# ]\n", + "\n", + "log_path = \"/home/shawn/NL2/nordpolder_van_delfgauw/\"\n", + "log_filenames = [\n", + "# log_path + \"platypus_20180213_022417.txt\",\n", + "# log_path + \"platypus_20180213_030409.txt\",\n", + "# log_path + \"platypus_20180213_035212.txt\",\n", + "# log_path + \"platypus_20180213_042357.txt\",\n", + "# log_path + \"platypus_20180213_054659.txt\",\n", + "# log_path + \"platypus_20180213_060554.txt\",\n", + "# log_path + \"platypus_20180213_062547.txt\",\n", + "# log_path + \"platypus_20180213_065743.txt\",\n", + "# log_path + \"platypus_20180213_073112.txt\",\n", + "# log_path + \"platypus_20180213_074623.txt\",\n", + "# log_path + \"platypus_20180213_082044.txt\",\n", + "# log_path + \"platypus_20180213_085649.txt\",\n", + " log_path + \"platypus_20180215_040330.txt\"\n", + "]\n", "\n", - "log_path = \"/home/shawn/all/\"\n", + "log_path = \"/home/shawn/NL2/grokeneveldse_polder/grokeneveldse_polder_feb_2018/\"\n", "log_filenames = glob.glob(log_path+'/'+\"*.txt\")\n", "\n", - "csv_output_filename = \"Nov2017ShawnNetherlands\"\n", - "log_ext = \".txt\"\n", + "# log_path = \"/home/shawn/NL1/all_nov_2017/\"\n", + "# log_filenames = glob.glob(log_path+'/'+\"*.txt\")\n", + "\n", + "csv_output_filename = \"grokeneveldse_polderNov2018Netherlands\"\n", "\n", - "#data = platypus.io.logs.load(log_path + log_filename + log_ext)\n", + "# data = platypus.io.logs.load(log_path + log_filename + log_ext)\n", "data = platypus.io.logs.merge_files(log_filenames)\n", "\n", + "\n", "if \"ES2\" in data:\n", " print \"ES2 sensor is present. Trimming all data within EC = 0 time windows\\n\"\n", " # find all time windows where EC is exactly 0\n", " ES2_data = data[\"ES2\"]\n", " values = ES2_data[\"ec\"].values\n", - " #ec_eq_zero_indices = np.where(values == 0)[0]\n", - " ec_eq_zero_indices = np.where( (values < 200) )[0]\n", + "# ec_eq_zero_indices = np.where(values == 0)[0]\n", + " ec_eq_zero_indices = np.where( (values < 50) | (values > 2000) )[0]\n", + "# ec_eq_zero_indices = np.where(values < 50)[0]\n", " windows = list()\n", " windows.append([ec_eq_zero_indices[0]])\n", " left = ec_eq_zero_indices[0]\n", @@ -123,29 +155,31 @@ " data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])]\n", "else:\n", " print \"No ES2 sensor present. No trimming will be performed.\"\n", - " \n", - "\"\"\"\n", - "if \"ATLAS_PH\" in data:\n", - " print \"pH sensor is present. Trimming all data within pH < 6 time windows\\n\"\n", - " # find all time windows where pH is less than 6\n", - " pH_data = data[\"ATLAS_PH\"]\n", - " values = pH_data[\"ph\"].values\n", - " pH_lt_6_indices = np.where( (values < 6) | (values > 8.5))[0]\n", - " windows = list()\n", - " windows.append([pH_lt_6_indices[0]])\n", - " left = pH_lt_6_indices[0]\n", - " for ii in range(1, pH_lt_6_indices.shape[0]):\n", - " i = pH_lt_6_indices[ii]\n", - " if i - left > 5:\n", - " windows[-1].append(left)\n", - " windows.append([i])\n", - " left = i\n", - " windows[-1].append(pH_lt_6_indices[-1])\n", - " for window in windows:\n", - " time_window = [pH_data[\"ph\"].index.values[window[0]], pH_data[\"ph\"].index.values[window[1]]]\n", - " for k in data:\n", - " data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])]\n", - "\"\"\"\n", + "\n", + "\n", + "\n", + "# if \"ATLAS_PH\" in data:\n", + "# print \"pH sensor is present. Trimming all data within pH < 6 time windows\\n\"\n", + "# # find all time windows where pH is less than 6\n", + "# pH_data = data[\"ATLAS_PH\"]\n", + "# values = pH_data[\"ph\"].values\n", + "# # pH_lt_6_indices = np.where( (values < 6) | (values > 8.5))[0]\n", + "# pH_lt_6_indices = np.where( (values < 6.5) )[0]\n", + "# windows = list()\n", + "# windows.append([pH_lt_6_indices[0]])\n", + "# left = pH_lt_6_indices[0]\n", + "# for ii in range(1, pH_lt_6_indices.shape[0]):\n", + "# i = pH_lt_6_indices[ii]\n", + "# if i - left > 5:\n", + "# windows[-1].append(left)\n", + "# windows.append([i])\n", + "# left = i\n", + "# windows[-1].append(pH_lt_6_indices[-1])\n", + "# for window in windows:\n", + "# time_window = [pH_data[\"ph\"].index.values[window[0]], pH_data[\"ph\"].index.values[window[1]]]\n", + "# for k in data:\n", + "# data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])]\n", + "\n", "\n", "# Define useful access variables.\n", "pose = data['pose']\n", @@ -162,26 +196,31 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "# Select the sensor and the name of the channel for that sensor.\n", - "# sensor_name = 'ATLAS_PH'\n", - "# sensor_channel = 'ph'\n", - "# sensor_units = \"pH\"\n", - "\n", "sensor_name = 'ATLAS_PH'\n", "sensor_channel = 'ph'\n", + "sensor_units = \"pH\"\n", + "\n", + "sensor_name = 'ES2'\n", + "sensor_channel = 'ec'\n", "sensor_units = 'Electrical Conductivity (uS/cm)'\n", "\n", + "# sensor_name = 'ES2'\n", + "# sensor_channel = 'temperature'\n", "# sensor_units = 'Temperature (C)'\n", - "#sensor_units = 'Dissolved Oxygen (mg/L)'" + "\n", + "# sensor_name = 'ATLAS_DO'\n", + "# sensor_channel = 'do'\n", + "# sensor_units = 'Dissolved Oxygen (mg/L)'" ] }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 16, "metadata": { "scrolled": true }, @@ -212,7 +251,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -224,36 +263,23 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 18, "metadata": { - "scrolled": true + "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "latitude 51.980245\n", - "longitude 4.258278\n", + "latitude 51.980277\n", + "longitude 4.258412\n", "dtype: float64\n", - "latitude 51.994891\n", - "longitude 4.291409\n", + "latitude 51.994885\n", + "longitude 4.290340\n", "dtype: float64\n", - "[1e-05, 1e-05]\n" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0mdata_estimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msensor\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'latitude'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'longitude'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msensor\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msensor_channel\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m \u001b[0mdata_zv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata_estimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_xy\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 29\u001b[0m \u001b[0mdata_zv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata_zv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/home/shawn/src/analytics/venv/local/lib/python2.7/site-packages/sklearn/neighbors/regression.pyc\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m 290\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mweights\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 291\u001b[0m y_pred = np.array([np.mean(_y[ind, :], axis=0)\n\u001b[0;32m--> 292\u001b[0;31m for ind in neigh_ind])\n\u001b[0m\u001b[1;32m 293\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 294\u001b[0m y_pred = np.array([(np.average(_y[ind, :], axis=0,\n", - "\u001b[0;32m/home/shawn/src/analytics/venv/local/lib/python2.7/site-packages/numpy/core/fromnumeric.pyc\u001b[0m in \u001b[0;36mmean\u001b[0;34m(a, axis, dtype, out, keepdims)\u001b[0m\n\u001b[1;32m 2907\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2908\u001b[0m return _methods._mean(a, axis=axis, dtype=dtype,\n\u001b[0;32m-> 2909\u001b[0;31m out=out, **kwargs)\n\u001b[0m\u001b[1;32m 2910\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2911\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/home/shawn/src/analytics/venv/local/lib/python2.7/site-packages/numpy/core/_methods.pyc\u001b[0m in \u001b[0;36m_mean\u001b[0;34m(a, axis, dtype, out, keepdims)\u001b[0m\n\u001b[1;32m 58\u001b[0m \u001b[0;31m# Make this warning show up first\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 59\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrcount\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 60\u001b[0;31m \u001b[0mwarnings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwarn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Mean of empty slice.\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mRuntimeWarning\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstacklevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 61\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[0;31m# Cast bool, unsigned int, and int to float64 by default\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + "[1e-05, 1e-05]\n", + "Data min = 50.000000 Data max = 1767.000000\n" ] } ], @@ -298,7 +324,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 19, "metadata": { "scrolled": false }, @@ -307,7 +333,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "removing file: ./platypus_data_f5ac49ed-6ff4-49f2-b515-0c18c4a248cf.png\n" + "removing file: ./platypus_data_39efe574-e0e4-4275-9725-c022ef98ff09.png\n", + "removing file: ./platypus_data_59589915-d7f5-4605-b8b1-2b99f8341d7f.png\n", + "removing file: ./platypus_data_54d1e0c2-83d4-4bde-9374-1dfe81a9cbf7.png\n" ] }, { @@ -317,7 +345,10 @@ "/home/shawn/src/analytics/venv/lib/python2.7/site-packages/ipykernel_launcher.py:15: DeprecationWarning: `imsave` is deprecated!\n", "`imsave` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n", "Use ``imageio.imwrite`` instead.\n", - " from ipykernel import kernelapp as app\n" + " from ipykernel import kernelapp as app\n", + "/home/shawn/src/analytics/venv/lib/python2.7/site-packages/ipykernel_launcher.py:21: DeprecationWarning: `imsave` is deprecated!\n", + "`imsave` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n", + "Use ``imageio.imwrite`` instead.\n" ] } ], @@ -339,12 +370,20 @@ " scipy.misc.imsave(png_filename, data_rgb)\n", "\n", " # Create image overlay that references generated image.\n", + " \n", + " data_rgb2 = np.ones((data_shape[0], data_shape[1], 4), dtype=np.uint8) * 50\n", + " png_filename2 = './platypus_data_{:s}.png'.format(uuid.uuid4())\n", + " scipy.misc.imsave(png_filename2, data_rgb2)\n", + " \n", + " data_bounds2 = [(position.min() - [x * 500 for x in data_padding]).tolist(),\n", + " (position.max() + [x * 500 for x in data_padding]).tolist()]\n", + " io_blank = ImageOverlay(url=png_filename2, bounds=data_bounds2)\n", " io = ImageOverlay(url=png_filename, bounds=data_bounds)" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 20, "metadata": { "scrolled": false }, @@ -352,7 +391,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "faf94d69a0094da2b0179e36f986344b", + "model_id": "4d5c4498fc894dafafea4d8a8127f135", "version_major": 2, "version_minor": 0 }, @@ -372,7 +411,7 @@ "

\n" ], "text/plain": [ - "Map(center=[51.983079299809056, 4.288419291357442], layers=(TileLayer(attribution=u'Map data (c) OpenStreetMap contributors', options=[u'opacity', u'attribution', u'max_zoom', u'detect_retina', u'min_zoom', u'tile_size'], url=u'https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png'), ImageOverlay(bounds=[[51.98242187260349, 4.285285712486869], [51.98466966749721, 4.28902748295083]], options=[u'opacity', u'attribution'], url=u'./platypus_data_c40dd200-a93e-4ba2-9a7e-b3c193762c00.png')), layout=Layout(align_self=u'stretch', height=u'400px'), options=[u'keyboard_pan_offset', u'tap', u'attribution_control', u'max_zoom', u'min_zoom', u'bounce_at_zoom_limits', u'keyboard', u'scroll_wheel_zoom', u'dragging', u'inertia_max_speed', u'close_popup_on_click', u'zoom_control', u'box_zoom', u'double_click_zoom', u'tap_tolerance', u'zoom_start', u'keyboard_zoom_offset', u'inertia_deceleration', u'inertia', u'center', u'zoom', u'world_copy_jump', u'zoom_animation_threshold', u'touch_zoom'], zoom=14)" + "Map(basemap={'url': 'https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', 'max_zoom': 19, 'attribution': 'Map data (c) OpenStreetMap contributors'}, center=[51.98821600381582, 4.273220401404014], default_tiles=TileLayer(base=True, max_zoom=19, min_zoom=1, options=[u'attribution', u'tile_size', u'min_zoom', u'max_zoom', u'detect_retina']), layers=(TileLayer(base=True, max_zoom=19, min_zoom=1, options=[u'attribution', u'tile_size', u'min_zoom', u'max_zoom', u'detect_retina']), ImageOverlay(bounds=[[51.93027687210173, 4.208411660165766], [52.044884747375924, 4.340339612362805]], options=[u'attribution'], url=u'./platypus_data_80269d7c-3e75-422d-a1aa-4fc6fea7142d.png'), ImageOverlay(bounds=[[51.98017687210172, 4.2583116601657665], [51.99498474737593, 4.290439612362805]], options=[u'attribution'], url=u'./platypus_data_8ed61f17-ad09-414c-b667-33defd6c53c6.png')), options=[u'keyboard_pan_offset', u'tap', u'attribution_control', u'max_zoom', u'min_zoom', u'bounce_at_zoom_limits', u'keyboard', u'scroll_wheel_zoom', u'dragging', u'inertia_max_speed', u'close_popup_on_click', u'zoom_control', u'box_zoom', u'double_click_zoom', u'tap_tolerance', u'zoom_start', u'keyboard_zoom_offset', u'inertia_deceleration', u'inertia', u'center', u'zoom', u'world_copy_jump', u'zoom_animation_threshold', u'touch_zoom', u'basemap'], zoom=14)" ] }, "metadata": {}, @@ -380,9 +419,9 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+EAAABRCAYAAAC9m6cOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAADbVJREFUeJzt3X+sJWV9x/H3h92VFZSfWymgFgxU\ng4IrQovBKAq1qFRNqqKVij9qbWoq2pjGWuuvaqNJ4wq1IfEHoNuWgqStW9pCqYCmVEARVkCwUKsi\nXWEpuKgQ3V2+/WOeC+eevffcPfeeO3d3eb+SzZ1nnh/zzNwzd/Y7zzNzUlVIkiRJkqTFt9tSd0CS\nJEmSpEcLg3BJkiRJknpiEC5JkiRJUk8MwiVJkiRJ6olBuCRJkiRJPTEIlyRJkiSpJwbhkiRJkiT1\nxCBckiRJkqSeGIRLkiRJktST5eMUTg4reGAqNZw7Ij1O2Um3NezR1q8JmuThWUjbO0tbC2nbthb3\nlOmtrZqj7vT8jCg/nDeqbJesGZdnMpw/qvy2ecN15yo/v+2OW34hbe8o/Ri37Um29Wg4XuO1Pf/P\n+XD+ku5TDfRj9Km8bf6w2Q/PtulRdecqP1fZxao7nD/JffR4jbftxWxrnLrjlp1g3W2ya+blGdMj\nujGcP+4uLeTXtpiHejH7NdfHaQNcWlUnz5A1zVhBeBeAv3WWqitGND0qb9z8uepOsq2l2sdJtjWm\nwf9ZLBtzs8PpwfoLqTtcfyF1h8svpO5wemc9XovZ1qj0JI/9Qvu1kGO/TXrgT/LyrUN509O7Ldsy\nPXvF9PxlA+WXLR/OG6o7nL/bQF2G8rZJD7U1ovxC6i5uW/OvO1x//LqTbGv2/VhIP8ZvayGfiR3l\n87XQz8SO9/kaux9bh9JbBvqx9aGhPEamM/TnbFpXtozIAxhVdzg9Ttm5tmW/lq5fC6n7aOjXmHU3\nD+UPnMps3jp73ox1hza1eZblucrOlb+QusP5w3mTbGuS+wjwAVg1w+ptOB1dkiRJkqSeGIRLkiRJ\nktQTg3BJkiRJknpiEC5JkiRJUk8MwiVJkiRJ6olBuCRJkiRJPTEIlyRJkiSpJwbhkiRJkiT1xCBc\nkiRJkqSeGIRLkiRJktQTg3BJkiRJknpiEC5JkiRJUk8MwiVJkiRJ6olBuCRJkiRJPTEIlyRJkiSp\nJwbhkiRJkiT1xCBckiRJkqSeGIRLkiRJktQTg3BJkiRJknpiEC5JkiRJUk8MwiVJkiRJ6olBuCRJ\nkiRJPTEIlyRJkiSpJwbhkiRJkiT1xCBckiRJkqSeGIRLkiRJktQTg3BJkiRJknpiEC5JkiRJUk8M\nwiVJkiRJ6olBuCRJkiRJPTEIlyRJkiSpJ6mq7S+cXAKsWrzuSLu8VcA9S90JaRfl+SUtHs8vaXF4\nbu1a7qmqk+cqNFYQLmlhkny9qo5Z6n5IuyLPL2nxeH5Ji8Nz69HJ6eiSJEmSJPXEIFySJEmSpJ4Y\nhEv9+tRSd0DahXl+SYvH80taHJ5bj0I+Ey5JkiRJUk8cCZckSZIkqScG4dKEJVmW5PokF7f0oUmu\nSXJ7kguSPKat372lb2/5hyxlv6UdXZJ9klyU5NYktyR5TpL9klyW5Lb2c99WNknOaufXN5McvdT9\nl3ZkSd6Z5OYkNyU5P8lKr1/S/CQ5J8ndSW4aWDf29SrJ6a38bUlOX4p90eIwCJcm7wzgloH0x4A1\nVXUYcB/w5rb+zcB9bf2aVk7S7M4ELqmqpwHPpDvP3g18qaoOB77U0gAvBg5v/34XOLv/7ko7hyQH\nA28HjqmqZwDLgNfg9Uuar/OA4e+KHut6lWQ/4P3ArwK/Arx/KnDXzs8gXJqgJE8EXgp8pqUDvBC4\nqBX5HPCKtvzylqbln9jKSxqSZG/gecBnAarq51X1I6afR8Pn1+erczWwT5IDe+62tDNZDjw2yXJg\nD2ADXr+keamqrwD3Dq0e93r168BlVXVvVd0HXMa2gb12Ugbh0mR9Avgj4KGW3h/4UVVtaekfAAe3\n5YOBOwBa/qZWXtK2DgU2Aue2xz0+k2RP4ICq2tDK/BA4oC0/fH41g+eepAFVdSfwF8D36YLvTcB1\neP2SJmnc65XXsV2YQbg0IUlOAe6uquuWui/SLmg5cDRwdlU9C/gpj0zlA6C6r/vwKz+kMbUpri+n\nu9l1ELAnjrhJi8brlQzCpck5HnhZku8Cf0c3je9MumlFy1uZJwJ3tuU7gScBtPy9gf/rs8PSTuQH\nwA+q6pqWvoguKL9rapp5+3l3y3/4/GoGzz1J050E/E9VbayqzcDf013TvH5JkzPu9crr2C7MIFya\nkKr646p6YlUdQvdCm8ur6nXAFcArW7HTgS+25XUtTcu/vN0ZlTSkqn4I3JHkqW3VicC3mH4eDZ9f\nr29vnT0O2DQwDVDSdN8HjkuyR3u2e+r88volTc6416tLgRcl2bfNVnlRW6ddQPybKU1ekhOAd1XV\nKUmeQjcyvh9wPXBaVf0syUpgLfAsupd3vKaqvrNUfZZ2dElW07308DHAd4A30t1MvhB4MvA94NVV\ndW8LJD5JN6X2AeCNVfX1Jem4tBNI8kHgVGAL3bXqd+ieP/X6JY0pyfnACcAq4C66t5z/I2Ner5K8\nCXhPa/YjVXVun/uhxWMQLkmSJElST5yOLkmSJElSTwzCJUmSJEnqiUG4JEmSJEk9MQiXJEmSJKkn\nBuGSJEmSJPXEIFyStMNIsjXJDQP/3t3WX5nkmHm0tzrJS0bkH5PkrHn2dcY+JVmR5KNJbkvyjSRf\nTfLi+WxjhrZ/Ms96045DkpdNHdsRdT6U5KS2/I4ke4y5zSS5PMlec5Q7Lsk17fd9S5IPDOStSPKN\ncbY7yzb+vX3PriRJS275UndAkqQBD1bV6gm2txo4BviX4Ywky9t3sU76+8P/DDgQeEb7TuUDgOdP\neBvjmnYcqmodsG5Uhap630DyHcBf032H7fZ6CbC+qu6fo9zn6L4vd32SZcBTB/KeC1w1xjZnsxb4\nfeAjE2hLkqQFcSRckrRTSfKiNrr8jSRfSPK4tv7YJP+ZZH2Sa5PsDXwIOLWNsp6a5ANJ1ia5Clib\n5IQkF7f6j0tybpIbk3wzyW+29Wcn+XqSm5N8cI6+7QG8BfiDqvoZQFXdVVUXtvzXtvZvSvKxgXo/\nSfKR1verW+BOkkPbvt6Y5MMD5R/ud0t/MskbxjgOb2h19k7yvSS7tbp7JrmjjUCfl+SVSd4OHARc\nkeSKJG9K8omBbb8lyZoZDsfrgC+2MockuWmgzrsGRryfAGxox2prVX1roI2TgX9tdV7ffi/rk6xt\n685rv5+rk3ynHZdz2oj6eQPtrANeO+p3J0lSXwzCJUk7ksdm+nT0Uwczk6wC3gucVFVH041i/2GS\nxwAXAGdU1TOBk4CfAu8DLqiq1VV1QWvmiFZ/OCj7U2BTVR1ZVUcBl7f1f1JVxwBHAc9PctSI/h8G\nfH+m0d8kBwEfA15INzJ9bJJXtOw9gatb379CF8gDnAmcXVVH0gLVUcY8DlTVJuAGHhmpPwW4tKo2\nD5Q5C/hf4AVV9QLgQuA3kqxoRd4InDNDd44Hrpurz8Aa4NtJ/iHJW5OsHMh7AXBlkqfT/d5f2Pbr\njIEy+wLPAd5JF2yvAZ4OHJlkdduH+4Ddk+y/Hf2RJGlRGYRLknYkD7ZAcfVwwNgcRxdEX5XkBuB0\n4JfopjBvqKqvAVTV/VW1ZZZtrKuqB2dYfxLwV1OJFrgBvLo9l3w9XXB3xDz37Vjgyqra2Pr2N8Dz\nWt7PgamR7euAQ9ry8cD5bXntdmxjnOMw5QJg6mbHa1p6VlX1E7obFKckeRqwoqpunKHoflX147k6\nXFUfopsq/2/AbwGXACQ5GLi3qh6gu3Hxhaq6p9W5d6CJf6qqAm4E7qqqG6vqIeBmHjmOAHfTjehL\nkrSkfCZckrQzCXDZ8Ch2kiPHaOOn272x5FDgXcCxVXVfm+K8ckSV24EnJ9lrO56FHrS5BZIAW5l+\nfa4Zym9h+o30UX2ayzrgz5PsBzybR2YAjPIZ4D3ArcC5s5TZkmS3FhCP7G9V/TdwdpJPAxvbiPXJ\nwKXb0ZeftZ8PDSxPpQeP40pgppsvkiT1ypFwSdLO5Grg+CSHwcPPMP8y8G3gwCTHtvWPT7Ic+DHw\n+O1s+zLgbVOJ9jbtveiC9k3tOe2Rbzlvo7afBc5sU8NJ8gtJXgVcSzedfVV7AdlrgS/P0aer6Ean\noXvGesr3gCOS7J5kH+DEtn7s49BGtr9GN/X94qraOkOxafWr6hrgSXQj1+fPUH6qL09py3cBT0iy\nf5Ld6aa90/r40iRpycPpbkL8iIHnweluDLxqajp5u2Gw3Vr7vwh8d5x6kiQtBoNwSdKOZPiZ8I8O\nZlbVRuANwPlJvgl8FXhaVf2cbkr1XyZZTxdQrwSuoAtWt3m+fAYfBvZtL01bT/cM9Hq6aei3An/L\n9r2p+73ARuBb7WVkFwP3V9UG4N2tT+uB66rqi3O0dQbwtiQ3AgcPHIc76J7Nvqn9vL6tn+9xuAA4\njdmnon8KuCTJFQPrLgSuGpi2P+yfgRNavzbTvRzu2tanWwfK/TbdM+E30E25n7rZcFhV3drq30z3\nZvMvt/36+CzbnM2z6Z65n2tqviRJiy6PzH6TJEnaPu3t7Guq6kuz5B8IfL6qfm0ebT8XOK2qfm+B\n3Zxq70y6dwHM2FdJkvrkSLgkSdpuSfZJ8l90L9GbNahtI/+fTrLXuNuoqv+YVADe3GQALknaUTgS\nLkmSJElSTxwJlyRJkiSpJwbhkiRJkiT1xCBckiRJkqSeGIRLkiRJktQTg3BJkiRJknpiEC5JkiRJ\nUk/+H/DgIpMMtIAdAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+EAAABRCAYAAAC9m6cOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADx0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wcmMxLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvjNbHMQAAEBtJREFUeJzt3XmQZWV5x/HvjxkWQdmcqAgkkIBaIIgIBgujggRRiVgVF4gLqGU0IYpGk4AaNURTmKRETAxVKogSgiDROMEIEkWtEAHZhmFTiQtgkCUgqFjADE/+OG/DnTu9zJ3uPt3X+X6quvqedzvvPW+/3f2c855zU1VIkiRJkqT5t9FCd0CSJEmSpA2FQbgkSZIkST0xCJckSZIkqScG4ZIkSZIk9cQgXJIkSZKknhiES5IkSZLUE4NwSZIkSZJ6YhAuSZIkSVJPDMIlSZIkSerJ0lEKJ7sU3DexNZw7zfYoZee6rWEbWr/m0Fwentm0PS5tzaZt25rfKdNbWzVD3TXzM0354bzpynabNenryQznT1d+7bzhujOVX7/9jlp+Nm0vln6M2vZctrUhHK/R2l7/n/Ph/AV9TzXQj+mn8tr5w6Y+PGtvT1d3pvIzlZ2vusP5c/kePV6j7Xs+2xql7qhl57DuWtk1+etJt6fpxnD+qG9pNsM2n4d6Pvs104/TrXB+VR0ySdYaRgrCuwD8TVNU3XiapqfLGzV/prpz2dZCvce5bGtEg/9ZLBlxt8Pbg/VnU3e4/mzqDpefTd3h7XE9XvPZ1nTbc3nsZ9uv2Rz7tbYHfiUvXT2Ut+b2RktWrZm98Zr5SwbKL1k6nDdUdzh/o4G6DOWttT3U1jTlZ1N3ftta/7rD9UevO5dtTf0+ZtOP0duazc/EYvn5mu3PxOL7+Rq5H6uHtlcN9GP1Q0N5TLudoV9na3Rl1TR5ANPVHd4epexM+7JfC9ev2dTdEPo1Yt0Hh/IHpjIPrp46b9K6Q7t6cIrXM5WdKX82dYfzh/Pmsq25fI8A74dlkySvxeXokiRJkiT1xCBckiRJkqSeGIRLkiRJktQTg3BJkiRJknpiEC5JkiRJUk8MwiVJkiRJ6olBuCRJkiRJPTEIlyRJkiSpJwbhkiRJkiT1xCBckiRJkqSeGIRLkiRJktQTg3BJkiRJknpiEC5JkiRJUk8MwiVJkiRJ6olBuCRJkiRJPTEIlyRJkiSpJwbhkiRJkiT1xCBckiRJkqSeGIRLkiRJktQTg3BJkiRJknpiEC5JkiRJUk8MwiVJkiRJ6olBuCRJkiRJPTEIlyRJkiSpJwbhkiRJkiT1xCBckiRJkqSeGIRLkiRJktQTg3BJkiRJknpiEC5JkiRJUk8MwiVJkiRJ6olBuCRJkiRJPUlVrXvh5Dxg2fx1Z4O0DLhzoTuhWXEMx59jOP4cw/HnGP5qcBzHn2M4/hzDhXNnVR0yU6GRgnDNvSSXVdU+C90PrT/HcPw5huPPMRx/juGvBsdx/DmG488xXPxcji5JkiRJUk8MwiVJkiRJ6olB+ML7+EJ3QLPmGI4/x3D8OYbjzzH81eA4jj/HcPw5houc94RLkiRJktQTr4RLkiRJktQTg/B5lGTHJBcmuS7JtUmOaenbJrkgyffa921aepJ8NMmNSa5OsvfCvgNNSLIkyZVJzm3bOye5pI3VWUk2aembtu0bW/5OC9lvdZJsneScJDckuT7Js5yH4yfJ29vv0muSnJlkM+fi4pbk1CS3J7lmIG3kuZfkyFb+e0mOXIj3sqGaYgz/rv0+vTrJF5JsPZB3XBvD7yR5wUD6IS3txiTH9v0+NnSTjeNA3juSVJJlbdu5uAhNNYZJ3tLm47VJ/nYg3bm4iBmEz69VwDuqajdgP+DoJLsBxwJfrapdga+2bYAXAru2rz8ETu6/y5rCMcD1A9sfAk6sql2Au4E3tPQ3AHe39BNbOS28k4DzquopwNPoxtJ5OEaSbA+8Fdinqp4KLAEOx7m42J0GDH9e6khzL8m2wPuA3waeCbxvInBXL05j7TG8AHhqVe0JfBc4DqD9j3M4sHur80/tJPYS4GN0Y7wbcEQrq/6cxtrjSJIdgYOBmwaSnYuL02kMjWGSA4DDgKdV1e7A37d05+IiZxA+j6rq1qq6or3+Gd0//tvTTZZPt2KfBl7aXh8GfKY6FwNbJ9mu525rSJIdgBcDn2zbAQ4EzmlFhsdwYmzPAZ7fymuBJNkKeA5wCkBVPVBVP8V5OI6WAo9KshTYHLgV5+KiVlXfBO4aSh517r0AuKCq7qqqu+kCwLWCCc2Pycawqr5SVava5sXADu31YcBnq+r+qvoBcCNdsPZM4Maq+n5VPQB8tpVVT6aYi9CdpPxzYPAhUc7FRWiKMfwj4ISqur+Vub2lOxcXOYPwnrSlkE8HLgEeX1W3tqyfAI9vr7cHbh6odktL08L6CN0fqIfa9mOBnw78AzI4Tg+PYcu/p5XXwtkZuAP4VLpbCj6ZZAuch2Olqn5Md4b/Jrrg+x7gcpyL42jUueecXNxeD3y5vXYMx0iSw4AfV9WKoSzHcXw8CfiddtvVN5Ls29Idw0XOILwHSR4N/Cvwtqq6dzCvusfT+4j6RSrJocDtVXX5QvdF620psDdwclU9HfgFjyx/BZyH46AteTyM7qTKE4Et8ArM2HPujbck76a79e6Mhe6LRpNkc+BdwHsXui+alaXAtnS3vf4ZcLarvsaDQfg8S7IxXQB+RlV9viXfNrG8tX2fWDryY2DHgeo7tDQtnP2BlyT5Id2SnQPp7i/eui2JhTXH6eExbPlbAf/XZ4e1lluAW6rqkrZ9Dl1Q7jwcLwcBP6iqO6rqQeDzdPPTuTh+Rp17zslFKMlRwKHAq+qRz7t1DMfHb9Gd1FzR/sfZAbgiyRNwHMfJLcDn260Dl9Kt2lyGY7joGYTPo3Ym6hTg+qr68EDWcmDiiZJHAl8cSH9teyrlfsA9A0v2tACq6riq2qGqdqJ7wMXXqupVwIXAy1qx4TGcGNuXtfJe5VlAVfUT4OYkT25Jzweuw3k4bm4C9kuyefvdOjGOzsXxM+rcOx84OMk2bUXEwS1NCyTJIXS3ab2kqu4byFoOHJ7u0wl2pnuw16XAt4Fd032awSZ0f0+X991vPaKqVlbV46pqp/Y/zi3A3u1vpnNxfPwbcABAkicBmwB34lxc9JbOXESzsD/wGmBlkqta2ruAE+iWi7wB+BHwipb3H8CL6B6ecB/wun67qxH8BfDZJB8ArqQ99Kt9Pz3JjXQPzzh8gfqnNb0FOKP9wfk+3dzaCOfh2KiqS5KcA1xBt/z1SuDjwJdwLi5aSc4EngcsS3IL3ZOVR/obWFV3Jflrun8eAY6vqskeMKV5MMUYHgdsClzQVr5eXFVvrqprk5xNd4JsFXB0Va1u7fwJXcC2BDi1qq7t/c1swCYbx6o6ZYrizsVFaIq5eCpwarqPLXsAOLKdcHYuLnLxwoAkSZIkSf1wObokSZIkST0xCJckSZIkqScG4ZIkSZIk9cQgXJIkSZKknhiES5IkSZLUE4NwSdKikWR1kqsGvo5t6V9Pss96tLdXkhdNk79Pko+uZ18n7VOSjZOckOR7Sa5I8q0kL1yffUzS9s/Xs94axyHJSyaO7TR1jk9yUHv9tiSbj7jPJPlaki1nKLdfkkvaeF+f5P0DeRsnuWKU/U6xj/9sn2ssSdKC83PCJUmLyS+raq85bG8vYB+6z71dQ5KlVXUZcNkc7g/gr4HtgKdW1f1JHg88d473Mao1jkNVLQeWT1ehqt47sPk24J/pPjN4Xb0IWFFV985Q7tPAK6pqRZIlwJMH8p4NXDTCPqdyOvDHwAfnoC1JkmbFK+GSpLGS5OB2dfmKJJ9L8uiWvm+S/06yIsmlSbYCjgde2a6yvjLJ+5OcnuQi4PQkz0tybqv/6CSfSrIyydVJfr+ln5zksiTXJvmrGfq2OfBG4C1VdT9AVd1WVWe3/CNa+9ck+dBAvZ8n+WDr+8UtcCfJzu29rkzygYHyD/e7bf9jkqNGOA5HtTpbJflRko1a3S2S3NyuQJ+W5GVJ3go8EbgwyYVJXp/kIwP7fmOSEyc5HK8CvtjK7JTkmoE67xy44v044NZ2rFZX1XUDbRwCfLnVeW0blxVJTm9pp7XxuTjJ99txObVdUT9toJ3lwBHTjZ0kSX0xCJckLSaPyprL0V85mJlkGfAe4KCq2pvuKvafJtkEOAs4pqqeBhwE/AJ4L3BWVe1VVWe1ZnZr9YeDsr8E7qmqPapqT+BrLf3dVbUPsCfw3CR7TtP/XYCbJrv6m+SJwIeAA+muTO+b5KUtewvg4tb3b9IF8gAnASdX1R60QHU6Ix4Hquoe4CoeuVJ/KHB+VT04UOajwP8CB1TVAcDZwO8l2bgVeR1w6iTd2R+4fKY+AycC30nyhSRvSrLZQN4BwNeT7E437ge293XMQJltgGcBb6cLtk8Edgf2SLJXew93A5smeew69EeSpHllEC5JWkx+2QLFvYYDxmY/uiD6oiRXAUcCv0G3hPnWqvo2QFXdW1WrptjH8qr65STpBwEfm9hogRvAK9p9yVfSBXe7red72xf4elXd0fp2BvCclvcAMHFl+3Jgp/Z6f+DM9vr0ddjHKMdhwlnAxMmOw9v2lKrq53QnKA5N8hRg46paOUnRbavqZzN1uKqOp1sq/xXgD4DzAJJsD9xVVffRnbj4XFXd2ercNdDEv1dVASuB26pqZVU9BFzLI8cR4Ha6K/qSJC0o7wmXJI2TABcMX8VOsscIbfxinXeW7Ay8E9i3qu5uS5w3m6bKjcCvJ9lyHe6FHvRgCyQBVrPm3+eapPwq1jyRPl2fZrIc+Jsk2wLP4JEVANP5JPAu4AbgU1OUWZVkoxYQT9vfqvof4OQknwDuaFesDwHOX4e+3N++PzTwemJ78DhuBkx28kWSpF55JVySNE4uBvZPsgs8fA/zk4DvANsl2belPybJUuBnwGPWse0LgKMnNtrTtLekC9rvafdpT/uU83bV9hTgpLY0nCS/luTlwKV0y9mXtQeQHQF8Y4Y+XUR3dRq6e6wn/AjYLcmmSbYGnt/SRz4O7cr2t+mWvp9bVasnKbZG/aq6BNiR7sr1mZOUn+jLb7bXtwGPS/LYJJvSLXun9fHFSdI2d6U7CfFTBu4Hpzsx8PKJ5eTthME6a+0/AfjhKPUkSZoPBuGSpMVk+J7wEwYzq+oO4CjgzCRXA98CnlJVD9Atqf6HJCvoAurNgAvpgtW17i+fxAeAbdpD01bQ3QO9gm4Z+g3Av7BuT+p+D3AHcF17GNm5wL1VdStwbOvTCuDyqvriDG0dAxydZCWw/cBxuJnu3uxr2vcrW/r6HoezgFcz9VL0jwPnJblwIO1s4KKBZfvDvgQ8r/XrQbqHw13a+nTDQLnX0N0TfhXdkvuJkw27VNUNrf61dE82/0Z7Xx+eYp9TeQbdPfczLc2XJGne5ZHVb5IkSeumPZ39xKr66hT52wGfqarfXY+2nw28uqrePMtuTrR3Et2zACbtqyRJffJKuCRJWmdJtk7yXbqH6E0Z1LYr/59IsuWo+6iq/5qrALy5xgBckrRYeCVckiRJkqSeeCVckiRJkqSeGIRLkiRJktQTg3BJkiRJknpiEC5JkiRJUk8MwiVJkiRJ6olBuCRJkiRJPfl/gni0q6dxzigAAAAASUVORK5CYII=\n", "text/plain": [ - "" + "
" ] }, "metadata": {}, @@ -397,6 +436,7 @@ "zoom = 14\n", "m = Map(center=center, zoom=zoom, height='1300px')\n", "if sensor_name in data:\n", + " m += io_blank\n", " m += io # Add image overlay\n", "if sensor_name not in data:\n", " m += pl # Add vehicle trail, but only if there isn't heatmap data to look at\n", diff --git a/setup.py b/setup.py index 50da608..43124cb 100644 --- a/setup.py +++ b/setup.py @@ -35,12 +35,17 @@ ] }, install_requires=[ + 'pillow', 'pandas', 'pymongo', 'pyserial', 'six', 'scipy', - 'utm' + 'utm', + 'matplotlib', + 'jupyter', + 'ipyleaflet', + 'sklearn' ], test_suite="tests", ) From 5893b0a19024d043f074f6fc9d3b019d92a0dfd8 Mon Sep 17 00:00:00 2001 From: Shawn Hanna Date: Sat, 9 Jun 2018 13:59:59 -0400 Subject: [PATCH 26/43] added support for intcatch unstable logs (I am calling them v4_3_0) --- src/platypus/io/logs.py | 114 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 2 deletions(-) diff --git a/src/platypus/io/logs.py b/src/platypus/io/logs.py index 66aa1a7..6d49f8f 100644 --- a/src/platypus/io/logs.py +++ b/src/platypus/io/logs.py @@ -94,6 +94,17 @@ Defines dataframe field names for known data types in v4.2.0 logfiles. """ +_DATA_FIELDS_v4_3_0 = { + 'BATTERY': ('voltage',), + 'EC_DECAGON': ('ec',), + 'T_DECAGON': ('temperature',), + 'DO_ATLAS': ('do',), + 'PH_ATLAS': ('ph',), +} +""" +Defines dataframe field names for known data types in v4.2.0 logfiles. +""" + def merge_files(filename_list): """ @@ -138,6 +149,92 @@ def read_around_sampler(logfile, pump_duration_seconds=4*60): return +def read_v4_3_0(logfile): + """ + Reads text logs from a Platypus vehicle server logfile. + + :param logfile: the logfile as an iterable + :type logfile: python file-like + :returns: a dict containing the data from this logfile + :rtype: {str: pandas.DataFrame} + """ + raw_data = collections.defaultdict(list) + start_time = datetime.datetime.utcfromtimestamp(0) + + for line in logfile: + # Extract each line fron the logfile and convert the timestamp. + time_offset_ms, level, message = line.split('\t', 2) + + # Compute the timestamp for each log entry. + time_offset = datetime.timedelta(milliseconds=int(time_offset_ms)) + timestamp = start_time + time_offset + + # Try to parse the log as a JSON object. + try: + entry = json.loads(message) + except ValueError as e: + raise ValueError( + "Aborted after invalid JSON log message '{:s}': {:s}" + .format(message, e)) + + # If the line is a datetime, compute subsequent timestamps from this. + # We assume that "date" and "time" are always together in the entry. + if 'date' in entry: + timestamp = datetime.datetime.utcfromtimestamp( + entry['time'] / 1000.) + start_time = timestamp - time_offset + + # Extract appropriate data from each entry. + for k, v in six.viewitems(entry): + if k == 'pose': + zone = int(v['zone'][:-5]) + hemi = v['zone'].endswith('North') + raw_data[k].append([ + timestamp, + v['p'][0], + v['p'][1], + v['p'][2], + zone, hemi + ]) + elif k == 'sensor': + try: + + + raw_data[v['type']].append([timestamp] + [v['data']]) + except: + # do nothing + None + else: + pass + + # Convert the list data to pandas DataFrames and return them. + # For known types, clean up and label the data. + data = {} + + for k, v in six.viewitems(raw_data): + if k == 'pose': + data['pose'] = add_ll_to_pose_dataframe( + remove_outliers_from_pose_dataframe( + pandas.DataFrame(v, columns=('time', + 'easting', 'northing', + 'altitude', 'zone', 'hemi')) + .set_index('time') + ) + ) + elif k in _DATA_FIELDS_v4_3_0: + data[k] = (pandas.DataFrame( + v, columns=('time',) + _DATA_FIELDS_v4_3_0[k]) + .set_index('time')) + else: + print 'other' + # For sensor types that we don't know how to handle, + # provide an unlabeled data frame. + data[k] = (pandas.DataFrame(v) + .rename(columns={0: 'time'}, copy=False) + .set_index('time')) + return data + + def read_v4_2_0(logfile): """ Reads text logs from a Platypus vehicle server logfile. @@ -187,6 +284,8 @@ def read_v4_2_0(logfile): ]) elif k == 'sensor': try: + + raw_data[v['type']].append([timestamp] + v['data']) except: # do nothing @@ -205,7 +304,7 @@ def read_v4_2_0(logfile): pandas.DataFrame(v, columns=('time', 'easting', 'northing', 'altitude', 'zone', 'hemi')) - .set_index('time') + .set_index('time') ) ) elif k in _DATA_FIELDS_v4_2_0: @@ -218,7 +317,6 @@ def read_v4_2_0(logfile): data[k] = (pandas.DataFrame(v) .rename(columns={0: 'time'}, copy=False) .set_index('time')) - return data @@ -407,6 +505,18 @@ def read_v4_0_0(logfile, filename): return data +def load_v4_3_0(filename, *args, **kwargs): + """ + Loads a log from a v4.2.0 server from a filename. + + :param filename: path to a log file + :type filename: string + :returns: a dict containing the data from this logfile + :rtype: {str: numpy.recarray} + """ + with open(filename, 'r') as logfile: + return read_v4_3_0(logfile) + def load_v4_2_0(filename, *args, **kwargs): """ Loads a log from a v4.2.0 server from a filename. From b6ca302c4357be71b1c2ccc4297604b4809a5757 Mon Sep 17 00:00:00 2001 From: Shawn Hanna Date: Sat, 9 Jun 2018 14:01:00 -0400 Subject: [PATCH 27/43] updatd interp notebook to handle ERM data --- notebooks/Data_Interpolation.ipynb | 178 +++++++++++------------------ 1 file changed, 67 insertions(+), 111 deletions(-) diff --git a/notebooks/Data_Interpolation.ipynb b/notebooks/Data_Interpolation.ipynb index 6f370a1..97c995e 100644 --- a/notebooks/Data_Interpolation.ipynb +++ b/notebooks/Data_Interpolation.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 3, + "execution_count": 56, "metadata": {}, "outputs": [], "source": [ @@ -25,7 +25,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 49, "metadata": { "scrolled": false }, @@ -34,14 +34,18 @@ "name": "stdout", "output_type": "stream", "text": [ - "set([u'ATLAS_PH', u'BATTERY', 'pose', u'ATLAS_DO', u'ES2'])\n", + "here: DO_ATLAS\n", + "here: BATTERY\n", + "here: EC_DECAGON\n", + "here: PH_ATLAS\n", + "here: T_DECAGON\n", "ES2 sensor is present. Trimming all data within EC = 0 time windows\n", "\n", "Available sensors/channels:\n", - " ATLAS_PH, ph\n", - " ATLAS_DO, do\n", - " ES2, ec\n", - " ES2, temperature\n" + " DO_ATLAS, do\n", + " EC_DECAGON, ec\n", + " PH_ATLAS, ph\n", + " T_DECAGON, temperature\n" ] } ], @@ -50,56 +54,7 @@ "\n", "log_ext = \".txt\"\n", "\n", - "log_path = \"/home/shawn/NL2/day1/\"\n", - "log_filenames = [\n", - "# log_path + \"platypus_20171117_034941.txt\",\n", - "# log_path + \"platypus_20171117_045142.txt\",\n", - "# log_path + \"platypus_20171117_052611.txt\",\n", - "# log_path + \"platypus_20171117_063759.txt\",\n", - "# log_path + \"platypus_20171117_083516.txt\",\n", - "# log_path + \"platypus_20171117_085402.txt\",\n", - "# log_path + \"platypus_20171117_092146.txt\",\n", - "# log_path + \"platypus_20171117_095701.txt\",\n", - "# log_path + \"platypus_20171117_102132.txt\",\n", - "# log_path + \"platypus_20171117_102340.txt\",\n", - "# log_path + \"platypus_20171117_103741.txt\",\n", - " ]\n", - "\n", - "# log_path = \"/home/shawn/day 2/\"\n", - "# log_filenames = [\n", - "# log_path + \"platypus_20171119_081603.txt\",\n", - "# # log_path + \"platypus_20171119_093819.txt\",\n", - "# # log_path + \"platypus_20171119_102933.txt\",\n", - "# # log_path + \"platypus_20171119_105652.txt\",\n", - "# # log_path + \"platypus_20171119_105709.txt\",\n", - "# # log_path + \"platypus_20171119_112232.txt\"\n", - "# ]\n", - "\n", - "# log_path = \"/home/shawn/day 3/\"\n", - "# log_filenames = [\n", - "# # log_path + \"platypus_20171120_040729.txt\",\n", - "# # log_path + \"platypus_20171120_052156.txt\",\n", - "# # log_path + \"platypus_20171120_065410.txt\",\n", - "# log_path + \"platypus_20171120_072029.txt\",\n", - "# ]\n", - "\n", - "# log_path = \"/home/shawn/day 4/\"\n", - "# log_filenames = [\n", - "# # log_path + \"platypus_20171121_045529.txt\",\n", - "# # log_path + \"platypus_20171121_053515.txt\",\n", - "# # log_path + \"platypus_20171121_073237.txt\",\n", - "# log_path + \"platypus_20171121_074700.txt\",\n", - "# # log_path + \"platypus_20171121_084718.txt\",\n", - "# # log_path + \"platypus_20171121_092302.txt\",\n", - "# # log_path + \"platypus_20171121_095421.txt\",\n", - "# # log_path + \"platypus_20171121_101351.txt\",\n", - "# # log_path + \"platypus_20171121_103427.txt\",\n", - "# # log_path + \"platypus_20171121_103725.txt\",\n", - "# # log_path + \"platypus_20171121_110202.txt\",\n", - "# # log_path + \"platypus_20171121_113356.txt\"\n", - "# ]\n", - "\n", - "log_path = \"/home/shawn/NL2/nordpolder_van_delfgauw/\"\n", + "log_path = \"/home/shawn/data/ERM/log_files/\"\n", "log_filenames = [\n", "# log_path + \"platypus_20180213_022417.txt\",\n", "# log_path + \"platypus_20180213_030409.txt\",\n", @@ -113,28 +68,28 @@ "# log_path + \"platypus_20180213_074623.txt\",\n", "# log_path + \"platypus_20180213_082044.txt\",\n", "# log_path + \"platypus_20180213_085649.txt\",\n", - " log_path + \"platypus_20180215_040330.txt\"\n", + " log_path + \"platypus_20180604_105536.txt\"\n", "]\n", "\n", - "log_path = \"/home/shawn/NL2/grokeneveldse_polder/grokeneveldse_polder_feb_2018/\"\n", - "log_filenames = glob.glob(log_path+'/'+\"*.txt\")\n", + "# log_path = \"/home/shawn/Downloads/data/day1\"\n", + "# log_filenames = glob.glob(log_path+'/'+\"*.txt\")\n", "\n", "# log_path = \"/home/shawn/NL1/all_nov_2017/\"\n", "# log_filenames = glob.glob(log_path+'/'+\"*.txt\")\n", "\n", - "csv_output_filename = \"grokeneveldse_polderNov2018Netherlands\"\n", + "csv_output_filename = \"ERM_2018_SF\"\n", "\n", "# data = platypus.io.logs.load(log_path + log_filename + log_ext)\n", "data = platypus.io.logs.merge_files(log_filenames)\n", "\n", "\n", - "if \"ES2\" in data:\n", + "if \"EC_DECAGON\" in data:\n", " print \"ES2 sensor is present. Trimming all data within EC = 0 time windows\\n\"\n", " # find all time windows where EC is exactly 0\n", - " ES2_data = data[\"ES2\"]\n", + " ES2_data = data[\"EC_DECAGON\"]\n", " values = ES2_data[\"ec\"].values\n", "# ec_eq_zero_indices = np.where(values == 0)[0]\n", - " ec_eq_zero_indices = np.where( (values < 50) | (values > 2000) )[0]\n", + " ec_eq_zero_indices = np.where( (values < 2000))[0]\n", "# ec_eq_zero_indices = np.where(values < 50)[0]\n", " windows = list()\n", " windows.append([ec_eq_zero_indices[0]])\n", @@ -158,13 +113,13 @@ "\n", "\n", "\n", - "# if \"ATLAS_PH\" in data:\n", + "# if \"PH_ATLAS\" in data:\n", "# print \"pH sensor is present. Trimming all data within pH < 6 time windows\\n\"\n", "# # find all time windows where pH is less than 6\n", - "# pH_data = data[\"ATLAS_PH\"]\n", + "# pH_data = data[\"PH_ATLAS\"]\n", "# values = pH_data[\"ph\"].values\n", "# # pH_lt_6_indices = np.where( (values < 6) | (values > 8.5))[0]\n", - "# pH_lt_6_indices = np.where( (values < 6.5) )[0]\n", + "# pH_lt_6_indices = np.where( (values < 6) )[0]\n", "# windows = list()\n", "# windows.append([pH_lt_6_indices[0]])\n", "# left = pH_lt_6_indices[0]\n", @@ -196,31 +151,31 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 50, "metadata": {}, "outputs": [], "source": [ "# Select the sensor and the name of the channel for that sensor.\n", - "sensor_name = 'ATLAS_PH'\n", + "sensor_name = 'PH_ATLAS'\n", "sensor_channel = 'ph'\n", "sensor_units = \"pH\"\n", "\n", - "sensor_name = 'ES2'\n", + "sensor_name = 'EC_DECAGON'\n", "sensor_channel = 'ec'\n", "sensor_units = 'Electrical Conductivity (uS/cm)'\n", "\n", - "# sensor_name = 'ES2'\n", + "# sensor_name = 'T_DECAGON'\n", "# sensor_channel = 'temperature'\n", "# sensor_units = 'Temperature (C)'\n", "\n", - "# sensor_name = 'ATLAS_DO'\n", - "# sensor_channel = 'do'\n", - "# sensor_units = 'Dissolved Oxygen (mg/L)'" + "sensor_name = 'DO_ATLAS'\n", + "sensor_channel = 'do'\n", + "sensor_units = 'Turbidty (NTU)'" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 51, "metadata": { "scrolled": true }, @@ -251,9 +206,18 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 52, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/shawn/Downloads/analytics-shawn_netherlands_save/analytics-shawn_netherlands_save/venv/lib/python2.7/site-packages/ipykernel_launcher.py:2: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n", + " \n" + ] + } + ], "source": [ "# Create a trail of the vehicle's path on the map.\n", "pl = Polyline(locations=position.as_matrix().tolist())\n", @@ -263,7 +227,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 53, "metadata": { "scrolled": false }, @@ -272,22 +236,22 @@ "name": "stdout", "output_type": "stream", "text": [ - "latitude 51.980277\n", - "longitude 4.258412\n", + "latitude 37.756384\n", + "longitude -122.382500\n", "dtype: float64\n", - "latitude 51.994885\n", - "longitude 4.290340\n", + "latitude 37.757332\n", + "longitude -122.380777\n", "dtype: float64\n", "[1e-05, 1e-05]\n", - "Data min = 50.000000 Data max = 1767.000000\n" + "Data min = 53.866667 Data max = 1126.130000\n" ] } ], "source": [ "## Add a data overlay for the map\n", - "data_padding = [0.0001, 0.0001] # degrees lat/lon\n", + "data_padding = [0.00001, 0.00001] # degrees lat/lon\n", "data_resolution = [0.00001, 0.00001] # degrees lat/lon\n", - "data_interpolation_radius = 0.0002 # degrees lat/lon\n", + "data_interpolation_radius = 0.00001 # degrees lat/lon\n", "data_bounds = [(position.min() - data_padding).tolist(),\n", " (position.max() + data_padding).tolist()]\n", "print position.min()\n", @@ -324,7 +288,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 54, "metadata": { "scrolled": false }, @@ -333,20 +297,20 @@ "name": "stdout", "output_type": "stream", "text": [ - "removing file: ./platypus_data_39efe574-e0e4-4275-9725-c022ef98ff09.png\n", - "removing file: ./platypus_data_59589915-d7f5-4605-b8b1-2b99f8341d7f.png\n", - "removing file: ./platypus_data_54d1e0c2-83d4-4bde-9374-1dfe81a9cbf7.png\n" + "removing file: ./platypus_data_4f9c525b-15cb-4c02-86b5-ff6068d9c0f5.png\n", + "removing file: ./platypus_data_4e7f86d5-1ae2-4a07-b950-ec10ec93a1b9.png\n", + "removing file: ./platypus_data_628d10c5-64a1-4ed1-9e9d-9ec81b3e3f87.png\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "/home/shawn/src/analytics/venv/lib/python2.7/site-packages/ipykernel_launcher.py:15: DeprecationWarning: `imsave` is deprecated!\n", + "/home/shawn/Downloads/analytics-shawn_netherlands_save/analytics-shawn_netherlands_save/venv/lib/python2.7/site-packages/ipykernel_launcher.py:15: DeprecationWarning: `imsave` is deprecated!\n", "`imsave` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n", "Use ``imageio.imwrite`` instead.\n", " from ipykernel import kernelapp as app\n", - "/home/shawn/src/analytics/venv/lib/python2.7/site-packages/ipykernel_launcher.py:21: DeprecationWarning: `imsave` is deprecated!\n", + "/home/shawn/Downloads/analytics-shawn_netherlands_save/analytics-shawn_netherlands_save/venv/lib/python2.7/site-packages/ipykernel_launcher.py:21: DeprecationWarning: `imsave` is deprecated!\n", "`imsave` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n", "Use ``imageio.imwrite`` instead.\n" ] @@ -383,7 +347,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 55, "metadata": { "scrolled": false }, @@ -391,27 +355,12 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4d5c4498fc894dafafea4d8a8127f135", + "model_id": "b66f7a07105b446a88823d2713173e3f", "version_major": 2, "version_minor": 0 }, - "text/html": [ - "

Failed to display Jupyter Widget of type Map.

\n", - "

\n", - " If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n", - " that the widgets JavaScript is still loading. If this message persists, it\n", - " likely means that the widgets JavaScript library is either not installed or\n", - " not enabled. See the Jupyter\n", - " Widgets Documentation for setup instructions.\n", - "

\n", - "

\n", - " If you're reading this message in another frontend (for example, a static\n", - " rendering on GitHub or NBViewer),\n", - " it may mean that your frontend doesn't currently support widgets.\n", - "

\n" - ], "text/plain": [ - "Map(basemap={'url': 'https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', 'max_zoom': 19, 'attribution': 'Map data (c) OpenStreetMap contributors'}, center=[51.98821600381582, 4.273220401404014], default_tiles=TileLayer(base=True, max_zoom=19, min_zoom=1, options=[u'attribution', u'tile_size', u'min_zoom', u'max_zoom', u'detect_retina']), layers=(TileLayer(base=True, max_zoom=19, min_zoom=1, options=[u'attribution', u'tile_size', u'min_zoom', u'max_zoom', u'detect_retina']), ImageOverlay(bounds=[[51.93027687210173, 4.208411660165766], [52.044884747375924, 4.340339612362805]], options=[u'attribution'], url=u'./platypus_data_80269d7c-3e75-422d-a1aa-4fc6fea7142d.png'), ImageOverlay(bounds=[[51.98017687210172, 4.2583116601657665], [51.99498474737593, 4.290439612362805]], options=[u'attribution'], url=u'./platypus_data_8ed61f17-ad09-414c-b667-33defd6c53c6.png')), options=[u'keyboard_pan_offset', u'tap', u'attribution_control', u'max_zoom', u'min_zoom', u'bounce_at_zoom_limits', u'keyboard', u'scroll_wheel_zoom', u'dragging', u'inertia_max_speed', u'close_popup_on_click', u'zoom_control', u'box_zoom', u'double_click_zoom', u'tap_tolerance', u'zoom_start', u'keyboard_zoom_offset', u'inertia_deceleration', u'inertia', u'center', u'zoom', u'world_copy_jump', u'zoom_animation_threshold', u'touch_zoom', u'basemap'], zoom=14)" + "TWFwKGJhc2VtYXA9eyd1cmwnOiAnaHR0cHM6Ly97c30udGlsZS5vcGVuc3RyZWV0bWFwLm9yZy97en0ve3h9L3t5fS5wbmcnLCAnbWF4X3pvb20nOiAxOSwgJ2F0dHJpYnV0aW9uJzogJ01hcCDigKY=\n" ] }, "metadata": {}, @@ -419,7 +368,7 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+EAAABRCAYAAAC9m6cOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADx0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wcmMxLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvjNbHMQAAEBtJREFUeJzt3XmQZWV5x/HvjxkWQdmcqAgkkIBaIIgIBgujggRRiVgVF4gLqGU0IYpGk4AaNURTmKRETAxVKogSgiDROMEIEkWtEAHZhmFTiQtgkCUgqFjADE/+OG/DnTu9zJ3uPt3X+X6quvqedzvvPW+/3f2c855zU1VIkiRJkqT5t9FCd0CSJEmSpA2FQbgkSZIkST0xCJckSZIkqScG4ZIkSZIk9cQgXJIkSZKknhiES5IkSZLUE4NwSZIkSZJ6YhAuSZIkSVJPDMIlSZIkSerJ0lEKJ7sU3DexNZw7zfYoZee6rWEbWr/m0Fwentm0PS5tzaZt25rfKdNbWzVD3TXzM0354bzpynabNenryQznT1d+7bzhujOVX7/9jlp+Nm0vln6M2vZctrUhHK/R2l7/n/Ph/AV9TzXQj+mn8tr5w6Y+PGtvT1d3pvIzlZ2vusP5c/kePV6j7Xs+2xql7qhl57DuWtk1+etJt6fpxnD+qG9pNsM2n4d6Pvs104/TrXB+VR0ySdYaRgrCuwD8TVNU3XiapqfLGzV/prpz2dZCvce5bGtEg/9ZLBlxt8Pbg/VnU3e4/mzqDpefTd3h7XE9XvPZ1nTbc3nsZ9uv2Rz7tbYHfiUvXT2Ut+b2RktWrZm98Zr5SwbKL1k6nDdUdzh/o4G6DOWttT3U1jTlZ1N3ftta/7rD9UevO5dtTf0+ZtOP0duazc/EYvn5mu3PxOL7+Rq5H6uHtlcN9GP1Q0N5TLudoV9na3Rl1TR5ANPVHd4epexM+7JfC9ev2dTdEPo1Yt0Hh/IHpjIPrp46b9K6Q7t6cIrXM5WdKX82dYfzh/Pmsq25fI8A74dlkySvxeXokiRJkiT1xCBckiRJkqSeGIRLkiRJktQTg3BJkiRJknpiEC5JkiRJUk8MwiVJkiRJ6olBuCRJkiRJPTEIlyRJkiSpJwbhkiRJkiT1xCBckiRJkqSeGIRLkiRJktQTg3BJkiRJknpiEC5JkiRJUk8MwiVJkiRJ6olBuCRJkiRJPTEIlyRJkiSpJwbhkiRJkiT1xCBckiRJkqSeGIRLkiRJktQTg3BJkiRJknpiEC5JkiRJUk8MwiVJkiRJ6olBuCRJkiRJPTEIlyRJkiSpJwbhkiRJkiT1xCBckiRJkqSeGIRLkiRJktQTg3BJkiRJknpiEC5JkiRJUk8MwiVJkiRJ6olBuCRJkiRJPUlVrXvh5Dxg2fx1Z4O0DLhzoTuhWXEMx59jOP4cw/HnGP5qcBzHn2M4/hzDhXNnVR0yU6GRgnDNvSSXVdU+C90PrT/HcPw5huPPMRx/juGvBsdx/DmG488xXPxcji5JkiRJUk8MwiVJkiRJ6olB+ML7+EJ3QLPmGI4/x3D8OYbjzzH81eA4jj/HcPw5houc94RLkiRJktQTr4RLkiRJktQTg/B5lGTHJBcmuS7JtUmOaenbJrkgyffa921aepJ8NMmNSa5OsvfCvgNNSLIkyZVJzm3bOye5pI3VWUk2aembtu0bW/5OC9lvdZJsneScJDckuT7Js5yH4yfJ29vv0muSnJlkM+fi4pbk1CS3J7lmIG3kuZfkyFb+e0mOXIj3sqGaYgz/rv0+vTrJF5JsPZB3XBvD7yR5wUD6IS3txiTH9v0+NnSTjeNA3juSVJJlbdu5uAhNNYZJ3tLm47VJ/nYg3bm4iBmEz69VwDuqajdgP+DoJLsBxwJfrapdga+2bYAXAru2rz8ETu6/y5rCMcD1A9sfAk6sql2Au4E3tPQ3AHe39BNbOS28k4DzquopwNPoxtJ5OEaSbA+8Fdinqp4KLAEOx7m42J0GDH9e6khzL8m2wPuA3waeCbxvInBXL05j7TG8AHhqVe0JfBc4DqD9j3M4sHur80/tJPYS4GN0Y7wbcEQrq/6cxtrjSJIdgYOBmwaSnYuL02kMjWGSA4DDgKdV1e7A37d05+IiZxA+j6rq1qq6or3+Gd0//tvTTZZPt2KfBl7aXh8GfKY6FwNbJ9mu525rSJIdgBcDn2zbAQ4EzmlFhsdwYmzPAZ7fymuBJNkKeA5wCkBVPVBVP8V5OI6WAo9KshTYHLgV5+KiVlXfBO4aSh517r0AuKCq7qqqu+kCwLWCCc2Pycawqr5SVava5sXADu31YcBnq+r+qvoBcCNdsPZM4Maq+n5VPQB8tpVVT6aYi9CdpPxzYPAhUc7FRWiKMfwj4ISqur+Vub2lOxcXOYPwnrSlkE8HLgEeX1W3tqyfAI9vr7cHbh6odktL08L6CN0fqIfa9mOBnw78AzI4Tg+PYcu/p5XXwtkZuAP4VLpbCj6ZZAuch2Olqn5Md4b/Jrrg+x7gcpyL42jUueecXNxeD3y5vXYMx0iSw4AfV9WKoSzHcXw8CfiddtvVN5Ls29Idw0XOILwHSR4N/Cvwtqq6dzCvusfT+4j6RSrJocDtVXX5QvdF620psDdwclU9HfgFjyx/BZyH46AteTyM7qTKE4Et8ArM2HPujbck76a79e6Mhe6LRpNkc+BdwHsXui+alaXAtnS3vf4ZcLarvsaDQfg8S7IxXQB+RlV9viXfNrG8tX2fWDryY2DHgeo7tDQtnP2BlyT5Id2SnQPp7i/eui2JhTXH6eExbPlbAf/XZ4e1lluAW6rqkrZ9Dl1Q7jwcLwcBP6iqO6rqQeDzdPPTuTh+Rp17zslFKMlRwKHAq+qRz7t1DMfHb9Gd1FzR/sfZAbgiyRNwHMfJLcDn260Dl9Kt2lyGY7joGYTPo3Ym6hTg+qr68EDWcmDiiZJHAl8cSH9teyrlfsA9A0v2tACq6riq2qGqdqJ7wMXXqupVwIXAy1qx4TGcGNuXtfJe5VlAVfUT4OYkT25Jzweuw3k4bm4C9kuyefvdOjGOzsXxM+rcOx84OMk2bUXEwS1NCyTJIXS3ab2kqu4byFoOHJ7u0wl2pnuw16XAt4Fd032awSZ0f0+X991vPaKqVlbV46pqp/Y/zi3A3u1vpnNxfPwbcABAkicBmwB34lxc9JbOXESzsD/wGmBlkqta2ruAE+iWi7wB+BHwipb3H8CL6B6ecB/wun67qxH8BfDZJB8ArqQ99Kt9Pz3JjXQPzzh8gfqnNb0FOKP9wfk+3dzaCOfh2KiqS5KcA1xBt/z1SuDjwJdwLi5aSc4EngcsS3IL3ZOVR/obWFV3Jflrun8eAY6vqskeMKV5MMUYHgdsClzQVr5eXFVvrqprk5xNd4JsFXB0Va1u7fwJXcC2BDi1qq7t/c1swCYbx6o6ZYrizsVFaIq5eCpwarqPLXsAOLKdcHYuLnLxwoAkSZIkSf1wObokSZIkST0xCJckSZIkqScG4ZIkSZIk9cQgXJIkSZKknhiES5IkSZLUE4NwSdKikWR1kqsGvo5t6V9Pss96tLdXkhdNk79Pko+uZ18n7VOSjZOckOR7Sa5I8q0kL1yffUzS9s/Xs94axyHJSyaO7TR1jk9yUHv9tiSbj7jPJPlaki1nKLdfkkvaeF+f5P0DeRsnuWKU/U6xj/9sn2ssSdKC83PCJUmLyS+raq85bG8vYB+6z71dQ5KlVXUZcNkc7g/gr4HtgKdW1f1JHg88d473Mao1jkNVLQeWT1ehqt47sPk24J/pPjN4Xb0IWFFV985Q7tPAK6pqRZIlwJMH8p4NXDTCPqdyOvDHwAfnoC1JkmbFK+GSpLGS5OB2dfmKJJ9L8uiWvm+S/06yIsmlSbYCjgde2a6yvjLJ+5OcnuQi4PQkz0tybqv/6CSfSrIyydVJfr+ln5zksiTXJvmrGfq2OfBG4C1VdT9AVd1WVWe3/CNa+9ck+dBAvZ8n+WDr+8UtcCfJzu29rkzygYHyD/e7bf9jkqNGOA5HtTpbJflRko1a3S2S3NyuQJ+W5GVJ3go8EbgwyYVJXp/kIwP7fmOSEyc5HK8CvtjK7JTkmoE67xy44v044NZ2rFZX1XUDbRwCfLnVeW0blxVJTm9pp7XxuTjJ99txObVdUT9toJ3lwBHTjZ0kSX0xCJckLSaPyprL0V85mJlkGfAe4KCq2pvuKvafJtkEOAs4pqqeBhwE/AJ4L3BWVe1VVWe1ZnZr9YeDsr8E7qmqPapqT+BrLf3dVbUPsCfw3CR7TtP/XYCbJrv6m+SJwIeAA+muTO+b5KUtewvg4tb3b9IF8gAnASdX1R60QHU6Ix4Hquoe4CoeuVJ/KHB+VT04UOajwP8CB1TVAcDZwO8l2bgVeR1w6iTd2R+4fKY+AycC30nyhSRvSrLZQN4BwNeT7E437ge293XMQJltgGcBb6cLtk8Edgf2SLJXew93A5smeew69EeSpHllEC5JWkx+2QLFvYYDxmY/uiD6oiRXAUcCv0G3hPnWqvo2QFXdW1WrptjH8qr65STpBwEfm9hogRvAK9p9yVfSBXe7red72xf4elXd0fp2BvCclvcAMHFl+3Jgp/Z6f+DM9vr0ddjHKMdhwlnAxMmOw9v2lKrq53QnKA5N8hRg46paOUnRbavqZzN1uKqOp1sq/xXgD4DzAJJsD9xVVffRnbj4XFXd2ercNdDEv1dVASuB26pqZVU9BFzLI8cR4Ha6K/qSJC0o7wmXJI2TABcMX8VOsscIbfxinXeW7Ay8E9i3qu5uS5w3m6bKjcCvJ9lyHe6FHvRgCyQBVrPm3+eapPwq1jyRPl2fZrIc+Jsk2wLP4JEVANP5JPAu4AbgU1OUWZVkoxYQT9vfqvof4OQknwDuaFesDwHOX4e+3N++PzTwemJ78DhuBkx28kWSpF55JVySNE4uBvZPsgs8fA/zk4DvANsl2belPybJUuBnwGPWse0LgKMnNtrTtLekC9rvafdpT/uU83bV9hTgpLY0nCS/luTlwKV0y9mXtQeQHQF8Y4Y+XUR3dRq6e6wn/AjYLcmmSbYGnt/SRz4O7cr2t+mWvp9bVasnKbZG/aq6BNiR7sr1mZOUn+jLb7bXtwGPS/LYJJvSLXun9fHFSdI2d6U7CfFTBu4Hpzsx8PKJ5eTthME6a+0/AfjhKPUkSZoPBuGSpMVk+J7wEwYzq+oO4CjgzCRXA98CnlJVD9Atqf6HJCvoAurNgAvpgtW17i+fxAeAbdpD01bQ3QO9gm4Z+g3Av7BuT+p+D3AHcF17GNm5wL1VdStwbOvTCuDyqvriDG0dAxydZCWw/cBxuJnu3uxr2vcrW/r6HoezgFcz9VL0jwPnJblwIO1s4KKBZfvDvgQ8r/XrQbqHw13a+nTDQLnX0N0TfhXdkvuJkw27VNUNrf61dE82/0Z7Xx+eYp9TeQbdPfczLc2XJGne5ZHVb5IkSeumPZ39xKr66hT52wGfqarfXY+2nw28uqrePMtuTrR3Et2zACbtqyRJffJKuCRJWmdJtk7yXbqH6E0Z1LYr/59IsuWo+6iq/5qrALy5xgBckrRYeCVckiRJkqSeeCVckiRJkqSeGIRLkiRJktQTg3BJkiRJknpiEC5JkiRJUk8MwiVJkiRJ6olBuCRJkiRJPfl/gni0q6dxzigAAAAASUVORK5CYII=\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+EAAABRCAYAAAC9m6cOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAACylJREFUeJzt3X+MZWV9x/H3hx0QpY3sgrHC0kgD2kAT0dBCU2u0EtCCQhq1pral1saa1NQabUNbUzCpDU2arjZNbIkUrViK2Rqx1WA3QqP9o5uA2IIgQmyVRWD5JRJtkV2+/eM+I3fP3pk7l3vnzNw771ey2fP8vN9z95ln5zvnnDupKiRJkiRJ0vo7YqMDkCRJkiRpqzAJlyRJkiSpJybhkiRJkiT1xCRckiRJkqSemIRLkiRJktQTk3BJkiRJknpiEi5JkiRJUk9MwiVJkiRJ6olJuCRJkiRJPVmapHNySsH3l0vd1lXKk/Sd9VxdWy2uGZrl2zPN3PMy1zRzO9f6fsn0NleNGXtoe1bp321bre+gWCOPR+m2r9b/8Lbu2HH9n9nrTtp/mrk3SxyTzj3LubbC+zXZ3M98nXfbN/ScaiiO1b+UD2/vWvntOby82thx/cf1Xa+x3fZZnqPv12SvvZ5zTTJ20r4zHHtYc40+HlleJYxu+6SnNM0/23q+1esZ17jldB98vqpeM6LpEBMl4YME/LdXGHrkKlOv1jZp+7ixs5xro85xlnNNaPg7i20Tvmy3PDx+mrHd8dOM7fafZmy3PK/v13rOtVp5lu/9tHFN894fVh7akpcOdtoOLR+x7cChzUce2r5tqP+2pW5bZ2y3/YihsXTaDit35lql/zRj13euZz62O37ysbOca+XzmCaOyeeaZk1slvU17ZrYfOtr4jgOdsoHhuI4+FSnjVXL6Wxnh4RyYJU2gNXGdsuT9B33Wsa1cXFNM3YrxDXh2Cc77UNfyjx5cOW2kWM7L/XkCsfj+o5rn2Zst73bNsu5ZnmOAJfB8SOqD+Pt6JIkSZIk9cQkXJIkSZKknpiES5IkSZLUE5NwSZIkSZJ6YhIuSZIkSVJPTMIlSZIkSeqJSbgkSZIkST0xCZckSZIkqScm4ZIkSZIk9cQkXJIkSZKknpiES5IkSZLUE5NwSZIkSZJ6YhIuSZIkSVJPTMIlSZIkSeqJSbgkSZIkST0xCZckSZIkqScm4ZIkSZIk9cQkXJIkSZKknpiES5IkSZLUE5NwSZIkSZJ6YhIuSZIkSVJPTMIlSZIkSeqJSbgkSZIkST0xCZckSZIkqScm4ZIkSZIk9cQkXJIkSZKknpiES5IkSZLUE5NwSZIkSZJ6YhIuSZIkSVJPTMIlSZIkSeqJSbgkSZIkST1JVa29c3I9cPz6haM5czzw0EYHIc2Qa1qLyHWtReOa1qJxTS+Oh6rqNeM6TZSES8OS3FRVZ250HNKsuKa1iFzXWjSuaS0a1/TW4+3okiRJkiT1xCRckiRJkqSemIRrGldsdADSjLmmtYhc11o0rmktGtf0FuMz4ZIkSZIk9cQr4ZIkSZIk9cQkXCMlOSnJjUluT/LVJO9q9TuS7ElyV/t7e6tPkr9KcneS/0ryso09A2llSbYluSXJv7TyyUn2tvV7bZKjWv2zWvnu1v7CjYxbGiXJsUl2J/lakjuS/Kx7teZZkne37z1uS3JNkqPdpzVvkvxdkv1Jbhuqm3hvTnJx639Xkos34lw0eybhWskB4D1VdRpwNvA7SU4DLgG+UFWnAl9oZYDXAqe2P28HPtx/yNKavQu4Y6j858CuqjoFeBR4W6t/G/Boq9/V+kmbzYeA66vqJ4GXMFjb7tWaS0lOBH4XOLOqfgrYBrwZ92nNn48C3d8XPdHenGQHcClwFvAzwKXLibvmm0m4Rqqq+6rqy+34cQbf1J0IXAh8rHX7GHBRO74Q+Psa+A/g2CQv6DlsaawkO4HzgY+0coBfAHa3Lt11vbzedwOvbv2lTSHJc4FXAFcCVNUPquo7uFdrvi0Bz06yBDwHuA/3ac2Zqvoi8EinetK9+TxgT1U9UlWPAns4PLHXHDIJ11jt1q6XAnuB51fVfa3pfuD57fhE4J6hYftanbTZfBD4A+CpVj4O+E5VHWjl4bX7w3Xd2h9r/aXN4mTgQeCq9ojFR5Icg3u15lRV3Qv8BfAtBsn3Y8DNuE9rMUy6N7tnLyiTcK0qyY8A/wT8XlV9d7itBh+t78fra24kuQDYX1U3b3Qs0owsAS8DPlxVLwW+x9O3NwLu1Zov7VbbCxn8gOkE4Bi88qcF5N68tZmEa0VJjmSQgH+iqj7Vqh9YvnWx/b2/1d8LnDQ0fGerkzaTnwNen+R/gH9kcHvjhxjc9rXU+gyv3R+u69b+XODhPgOWxtgH7Kuqva28m0FS7l6teXUO8N9V9WBVPQl8isHe7T6tRTDp3uyevaBMwjVSe57qSuCOqvrLoabPAMufzHgxcN1Q/a+3T3c8G3hs6HYbaVOoqj+sqp1V9UIGH/RzQ1W9BbgReEPr1l3Xy+v9Da2/P7XWplFV9wP3JHlxq3o1cDvu1Zpf3wLOTvKc9r3I8pp2n9YimHRv/jxwbpLt7S6Rc1ud5lzcpzRKkpcDXwJu5elnZ/+IwXPhnwR+HPgm8KaqeqT9R/nXDG4Z+z7w1qq6qffApTVK8krgvVV1QZKfYHBlfAdwC/CrVfVEkqOBjzP4TIRHgDdX1Tc2KmZplCRnMPigwaOAbwBvZfBDdvdqzaUk7wd+mcFvarkF+C0Gz8G6T2tuJLkGeCVwPPAAg085/zQT7s1JfpPB9+AAH6iqq/o8D60Pk3BJkiRJknri7eiSJEmSJPXEJFySJEmSpJ6YhEuSJEmS1BOTcEmSJEmSemISLkmSJElST0zCJUlagyTHJflK+3N/knuHyketcY6rk1w0ov6sJLtWGLMvybFJdiR5xzOIe2eS69rxOUkqyWuH2q9P8vIkn2nncneSx4bO7azlGIbGnJPk0+34oiR/MmlckiRtVSbhkiStQVU9XFVnVNUZwN8Au5bLVfWDceOTLK0y996qeveYKXYAEyfhwHuAK4bK9wB/PCKG17dzewdw49C57R0z/3XAL7Xf1yxJksYwCZckaQpJTknylaHyJUne147/PcmuJDcB72xdzktyc5KvL1+R7lxZfl6SPUm+muRvgbRxlwMvblenL0/yD0kuGHrda5Oc34ktwEXAnqHqLwNPJHnVLM6/qgr4EvCLs5hPkqRFZxIuSdL62lZVZ1bVB1v5JOCngdcBVyR5Vqf/+xlciT4d+BxwQqu/BLizXZ2+BLgS+A2AJNvbnNd35joF2D/iSv0HgPdNfWZPuwn4+RnOJ0nSwjIJlyRpfV3bKX+yqp6qqjsZ3Bp+aqf9FcDVAFV1HfD4CvPeAJye5DjgLW3eg50+LwAe7A6sqhuAZyc5e43nUGPq9vP0DwskSdIqTMIlSZrOAQ79/7T7bPT3OuVuQjsqwR2r3QZ+NfArDK6IXzWi2/+OiGfZn7L2q+EPA9uHyjuAh4bKR7fXkiRJY5iES5I0nfuBE5Jsbx9Odv6Y/m/MwIsY3Jp+V6f9iwwSa5K8DvjRVv/40PGyq4DfB55oV9a77gROHhVEVX0O+DHg9DHxAvwb8GstpiUGV95vHGp/EXDbGuaRJGnLMwmXJGkKVfV/wJ8xeC76X4Hbxwy5t/X9Z+DtI57XvhQ4J8ltwAXAt9vrPADcnOTWJJe3um8DX2f0VXCq6rvAPUlGJuIt7p1j4gW4DDgtyX8y+GC3O4BrhtpfBXx2DfNIkrTlZXA3myRJmjdJjgFuBV5SVSOfHU/yRuD0qrpsnWI4AfhoVZ27HvNLkrRovBIuSdIcSnIegyvSu1ZKwJvdwL51DOUk4L3rOL8kSQvFK+GSJEmSJPXEK+GSJEmSJPXEJFySJEmSpJ6YhEuSJEmS1BOTcEmSJEmSemISLkmSJElST0zCJUmSJEnqyf8DVpg+08em6FkAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] @@ -433,7 +382,7 @@ "# Create a map centered on this data log.\n", "center = [pose['latitude'].median(), pose['longitude'].median()]\n", "# print center\n", - "zoom = 14\n", + "zoom = 17\n", "m = Map(center=center, zoom=zoom, height='1300px')\n", "if sensor_name in data:\n", " m += io_blank\n", @@ -461,6 +410,13 @@ "m" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, From d84f005d3c8c0fd28a8a74b4d2613c6a51413a55 Mon Sep 17 00:00:00 2001 From: Shawn Hanna Date: Wed, 13 Jun 2018 19:19:14 -0400 Subject: [PATCH 28/43] started working on the file processing server pipeline for ERM --- image_server/css/css.css | 1 + image_server/image_server.py | 260 ++++++++++++++++++++++++++++++ image_server/index.html | 8 + image_server/templates/index.html | 61 +++++++ setup.py | 3 +- 5 files changed, 332 insertions(+), 1 deletion(-) create mode 100644 image_server/css/css.css create mode 100644 image_server/image_server.py create mode 100644 image_server/index.html create mode 100644 image_server/templates/index.html diff --git a/image_server/css/css.css b/image_server/css/css.css new file mode 100644 index 0000000..49ce431 --- /dev/null +++ b/image_server/css/css.css @@ -0,0 +1 @@ +#mapid { height: 250px; width: 250px;} \ No newline at end of file diff --git a/image_server/image_server.py b/image_server/image_server.py new file mode 100644 index 0000000..9808f83 --- /dev/null +++ b/image_server/image_server.py @@ -0,0 +1,260 @@ +from ipyleaflet import Map, ImageOverlay, Polyline +import matplotlib +import matplotlib.cm +from matplotlib import pyplot +import numpy as np +import numpy.lib.recfunctions +import scipy +import scipy.interpolate +import pandas +import platypus.io.logs +import os +import uuid +import glob +import flask +from flask import send_from_directory, render_template + +UPLOAD_FOLDER = '/home/shawn/data/ERM/log_files/' +ALLOWED_EXTENSIONS = set(['txt']) + +from flask import Flask +app = Flask(__name__, static_url_path='', ) + +app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER + +@app.route('/path/') +def send(path): + if (os.path.exists(path)): + print "sending file: " + path + return send_from_directory("", path) + else: + print path +" does not exist" + +@app.route('/logs/') +def show_log(log_file): + # Import the data from the specified logfile + + log_ext = ".txt" + + log_path = "/home/shawn/data/ERM/log_files/" + log_filenames = [ + log_path + log_file + ] + + csv_output_filename = "ERM_2018_SF" + + data = platypus.io.logs.merge_files(log_filenames) + + + if "EC_DECAGON" in data: + print "ES2 sensor is present. Trimming all data within EC = 0 time windows\n" + # find all time windows where EC is exactly 0 + ES2_data = data["EC_DECAGON"] + values = ES2_data["ec"].values + # ec_eq_zero_indices = np.where(values == 0)[0] + ec_eq_zero_indices = np.where( (values < 2000))[0] + # ec_eq_zero_indices = np.where(values < 50)[0] + windows = list() + windows.append([ec_eq_zero_indices[0]]) + left = ec_eq_zero_indices[0] + for ii in range(1, ec_eq_zero_indices.shape[0]): + i = ec_eq_zero_indices[ii] + if i - left > 5: + # there has been a jump in index, a new time window has started + windows[-1].append(left) + windows.append([i]) + left = i + windows[-1].append(ec_eq_zero_indices[-1]) + # print ec_eq_zero_indices + # print windows + for window in windows: + time_window = [ES2_data["ec"].index.values[window[0]], ES2_data["ec"].index.values[window[1]]] + for k in data: + data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])] + else: + print "No ES2 sensor present. No trimming will be performed." + + + + # if "PH_ATLAS" in data: + # print "pH sensor is present. Trimming all data within pH < 6 time windows\n" + # # find all time windows where pH is less than 6 + # pH_data = data["PH_ATLAS"] + # values = pH_data["ph"].values + # # pH_lt_6_indices = np.where( (values < 6) | (values > 8.5))[0] + # pH_lt_6_indices = np.where( (values < 6) )[0] + # windows = list() + # windows.append([pH_lt_6_indices[0]]) + # left = pH_lt_6_indices[0] + # for ii in range(1, pH_lt_6_indices.shape[0]): + # i = pH_lt_6_indices[ii] + # if i - left > 5: + # windows[-1].append(left) + # windows.append([i]) + # left = i + # windows[-1].append(pH_lt_6_indices[-1]) + # for window in windows: + # time_window = [pH_data["ph"].index.values[window[0]], pH_data["ph"].index.values[window[1]]] + # for k in data: + # data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])] + + + # Define useful access variables. + pose = data['pose'] + position = pose[['latitude', 'longitude']] + + # Print the available sensors and channels for this logfile. + print "Available sensors/channels:" + for s in data.keys(): + if s == 'pose' or s == 'BATTERY': + continue + for c in data[s].dtypes.keys(): + print " {:s}, {:s}".format(s, str(c)) + + # Select the sensor and the name of the channel for that sensor. + sensor_name = 'PH_ATLAS' + sensor_channel = 'ph' + sensor_units = "pH" + + sensor_name = 'EC_DECAGON' + sensor_channel = 'ec' + sensor_units = 'Electrical Conductivity (uS/cm)' + + # sensor_name = 'T_DECAGON' + # sensor_channel = 'temperature' + # sensor_units = 'Temperature (C)' + + sensor_name = 'DO_ATLAS' + sensor_channel = 'do' + sensor_units = 'Turbidty (NTU)' + + # Extract the pose timing and the sensor data of interest. + pose_times = pose.index.values.astype(np.float64) + + if sensor_name in data: + sensor = data[sensor_name] + sensor_times = sensor.index.values.astype(np.float64) + + # Linearly interpolate the position of the sensor at every sample. + sensor_pose_interpolator = scipy.interpolate.interp1d(pose_times, position, + axis=0, bounds_error=False) + + # Add the position information back to the sensor data. + sensor = sensor.join(pandas.DataFrame(sensor_pose_interpolator(sensor_times), sensor.index, + columns=('latitude', 'longitude'))) + + # print sensor data to csv file + sensor.to_csv(log_path + csv_output_filename + "__" + sensor_name + ".csv") + + # Remove columns that have NaN values (no pose information). + sensor_valid = np.all(np.isfinite(sensor), axis=1) + sensor = sensor[sensor_valid] + + # Create a trail of the vehicle's path on the map. + pl = Polyline(locations=position.as_matrix().tolist()) + pl.fill_opacity = 0.0 + pl.weight = 2 + + ## Add a data overlay for the map + data_padding = [0.00001, 0.00001] # degrees lat/lon + data_resolution = [0.00001, 0.00001] # degrees lat/lon + data_interpolation_radius = 0.00001 # degrees lat/lon + data_bounds = [(position.min() - data_padding).tolist(), + (position.max() + data_padding).tolist()] + + print position.min() + print position.max() + print data_bounds + print data_resolution + + # Create a rectangular grid of overlay points. + data_xv, data_yv = np.meshgrid( + np.arange(data_bounds[1][0], data_bounds[0][0], -data_resolution[0]), + np.arange(data_bounds[0][1], data_bounds[1][1], data_resolution[1]) + ) + data_shape = data_xv.shape + data_xy = np.vstack([data_xv.ravel(), data_yv.ravel()]).T + + if sensor_name in data: + # Create a radial-basis interpolator over the sensor dataset + # Then, query it at each point of the rectangular grid. + #from sklearn.neighbors import RadiusNeighborsClassifier + #data_estimator = RadiusNeighborsClassifier(radius=data_interpolation_radius, outlier_label=np.nan) + from sklearn.neighbors import RadiusNeighborsRegressor + data_estimator = RadiusNeighborsRegressor(radius=data_interpolation_radius) + + data_estimator.fit(sensor[['latitude','longitude']], sensor[sensor_channel].astype(np.float)) + data_zv = data_estimator.predict(data_xy) + data_zv = data_zv.reshape(data_shape).T + + # Normalize data from [0, 1) + data_max = data_zv[np.isfinite(data_zv)].max() + data_min = data_zv[np.isfinite(data_zv)].min() + print "Data min = {:f} Data max = {:f}".format(data_min, data_max) + NORMALIZER = data_max # 800 + data_zv = (data_zv - data_min) / (NORMALIZER - data_min) + + + + if sensor_name in data: + # Update a color map only at the points that have valid values. + data_rgb = np.zeros((data_shape[0], data_shape[1], 4), dtype=np.uint8) + data_rgb = matplotlib.cm.jet(data_zv) * 255 + data_rgb[:,:,3] = 255 * np.isfinite(data_zv) + + # Remove any old image files. + old_png_files = glob.glob('./*.png') + for old_png_file in old_png_files: + print "removing file: " + old_png_file + os.remove(old_png_file) + + png_filename_rgb = './platypus_data_{:s}.png'.format(uuid.uuid4()) + scipy.misc.imsave(png_filename_rgb, data_rgb) + + # Create image overlay that references generated image. + # makes an image w/ a slight opaque tint to it + data_overlay_tint = np.ones((data_shape[0], data_shape[1], 4), dtype=np.uint8) * 50 + data_overlay_tint_filename = './platypus_data_{:s}.png'.format(uuid.uuid4()) + scipy.misc.imsave(data_overlay_tint_filename, data_overlay_tint) + + data_bounds_tint = [(position.min() - [x * 500 for x in data_padding]).tolist(), + (position.max() + [x * 500 for x in data_padding]).tolist()] + io_blank = ImageOverlay(url=data_overlay_tint_filename, bounds=data_bounds_tint) + io = ImageOverlay(url=png_filename_rgb, bounds=data_bounds) + + # Create a map centered on this data log. + center = [pose['latitude'].median(), pose['longitude'].median()] + # print center + zoom = 17 + # m = Map(center=center, zoom=zoom, height='1300px') + # if sensor_name in data: + # m += io_blank + # m += io # Add image overlay + # if sensor_name not in data: + # m += pl # Add vehicle trail, but only if there isn't heatmap data to look at + + # Make a figure and axes with dimensions as desired. + fig = pyplot.figure(figsize=(15, 3)) + ax1 = fig.add_axes([0.05, 0.80, 0.9, 0.15]) + + if sensor_name in data: + # Set the colormap and norm to correspond to the data for which + # the colorbar will be used. + cmap = matplotlib.cm.jet + norm = matplotlib.colors.Normalize(vmin=data_min, vmax=NORMALIZER) + cb1 = matplotlib.colorbar.ColorbarBase(ax1, cmap=cmap, + norm=norm, + orientation='horizontal') + cb1.set_label(sensor_units) + + scale_bar_filename = './platypus_data_{:s}.png'.format(uuid.uuid4()) + pyplot.savefig(scale_bar_filename) + pyplot.close('all') + # pyplot.show() + return render_template('index.html', bar = scale_bar_filename, map_overlay = png_filename_rgb, log_filenames = log_filenames, center=center, zoom=zoom, data_bounds=data_bounds) + +def root_dir(): # pragma: no cover + return os.path.abspath(os.path.dirname(__file__)) + +if __name__ == '__main__': + main() diff --git a/image_server/index.html b/image_server/index.html new file mode 100644 index 0000000..97c999f --- /dev/null +++ b/image_server/index.html @@ -0,0 +1,8 @@ + + + +

Hello {{ name }}!

+ + + + \ No newline at end of file diff --git a/image_server/templates/index.html b/image_server/templates/index.html new file mode 100644 index 0000000..2d48c32 --- /dev/null +++ b/image_server/templates/index.html @@ -0,0 +1,61 @@ + + + + + Quick Start - Leaflet + + + + + + + + + + + + + + + + +

{{ log_filenames }}!

+ +
+ + + + + + + + diff --git a/setup.py b/setup.py index 43124cb..e6409b1 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,8 @@ 'matplotlib', 'jupyter', 'ipyleaflet', - 'sklearn' + 'sklearn', + 'flask' ], test_suite="tests", ) From e8ba77d71b796db6c55fb50721cf5fde699c0a03 Mon Sep 17 00:00:00 2001 From: Shawn Hanna Date: Tue, 19 Jun 2018 07:31:24 -0400 Subject: [PATCH 29/43] some changes to the flask image server --- image_server/data_processor.py | 319 +++++++++++++++++++++++++++++ image_server/image_server.py | 60 +++--- image_server/templates/index.html | 10 +- notebooks/Data_Interpolation.ipynb | 39 ++-- 4 files changed, 374 insertions(+), 54 deletions(-) create mode 100644 image_server/data_processor.py diff --git a/image_server/data_processor.py b/image_server/data_processor.py new file mode 100644 index 0000000..05a3390 --- /dev/null +++ b/image_server/data_processor.py @@ -0,0 +1,319 @@ +from ipyleaflet import Map, ImageOverlay, Polyline + +from sklearn.neighbors import RadiusNeighborsRegressor +import matplotlib +import matplotlib.cm +from matplotlib import pyplot +import numpy as np +import numpy.lib.recfunctions +import scipy +import scipy.interpolate +import pandas +import platypus.io.logs +import os +import glob +import sys + +def sensor_id_to_name(id): + if (id == 0): + # Select the sensor and the name of the channel for that sensor. + sensor_name = 'PH_ATLAS' + sensor_channel = 'ph' + sensor_units = "pH" + elif (id == 1): + sensor_name = 'EC_DECAGON' + sensor_channel = 'ec' + sensor_units = 'Electrical Conductivity (uS/cm)' + elif (id == 2): + sensor_name = 'T_DECAGON' + sensor_channel = 'temperature' + sensor_units = 'Temperature (C)' + elif (id == 3): + sensor_name = 'DO_ATLAS' + sensor_channel = 'do' + sensor_units = 'Turbidty (NTU)' + + return (sensor_name, sensor_channel, sensor_units) + +def generate_overlay(log_file, sensor_id, min_ec): + # Import the data from the specified logfile + + log_ext = ".txt" + + log_path = "/home/shawn/data/ERM/log_files/" + + if (os.path.exists(log_path + log_file + log_ext) == False): + print "File doesn't exist: " + log_path + log_file + log_ext + log_ext = ".txt.incomplete" + + if (os.path.exists(log_path + log_file + log_ext) == False): + print "Error. log does not exist: " + log_path + log_file + log_ext + return False + + log_filenames = [ + log_path + log_file + log_ext + ] + + + + data = platypus.io.logs.merge_files(log_filenames) + + (sensor_name, sensor_channel, sensor_units) = sensor_id_to_name(sensor_id) + + # Define useful access variables. + pose = data['pose'] + position = pose[['latitude', 'longitude']] + + data_boundaries = [ [37.755690, -122.381139], [37.757928, -122.380338]] + + if "T_DECAGON" in data: + print "Temperature sensor is present. Trimming all data where temperature is changing a lot in a given time windows\n" + # find all time windows where EC is exactly 0 + + # print ES2_data + T_data = data["T_DECAGON"] + values = T_data["temperature"].values + + dtemp_dt_limit = 0.5/60.0 + + # pose_lat_vals = position["latitude"].values + # pose_lon_vals = position["longitude"].values + stddevs = [] + zero_indices = [] + for i, x in enumerate(values): + zero_indices.append(0) + stddevs.append(0) + if (i + 60 < len(values)): + vals = [] + for x in xrange(i, i+60): + vals.append(values[x]) + vals = np.array(vals) + stddev = vals.std() + stddevs[i] = stddev + # if (stddev > 0.1): + # print "@ i = " + str(i)+", time = " + str(T_data["temperature"].index[i]) + " - std dev: " + str(stddev) + + for i, x in enumerate(stddevs): + # print i + if (x > 0.1): + # print x + for y in xrange(i, i+60): + zero_indices[y] = 1 + + # print zero_indices + + # ec_eq_zero_indices = np.where(values == 0)[0] + ec_eq_zero_indices = np.where( (np.array(zero_indices) == 1) )[0] # | out_of_bouds_lat | out_of_bouds_lon )[0] + # print ec_eq_zero_indices + # ec_eq_zero_indices = np.where(values < 50)[0] + windows = list() + windows.append([ec_eq_zero_indices[0]]) + left = ec_eq_zero_indices[0] + for ii in range(1, ec_eq_zero_indices.shape[0]): + i = ec_eq_zero_indices[ii] + if i - left > 5: + # there has been a jump in index, a new time window has started + windows[-1].append(left) + windows.append([i]) + left = i + windows[-1].append(ec_eq_zero_indices[-1]) + # print ec_eq_zero_indices + # print windows + for window in windows: + print "window: " + str(window) + time_window = [T_data["temperature"].index.values[window[0]], T_data["temperature"].index.values[window[1]]] + for k in data: + data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])] + print T_data["temperature"].values[np.where( (np.array(zero_indices) ==1) )[0]] + else: + print "No ES2 sensor present. No trimming will be performed." + + if "EC_DECAGON" in data: + print "ES2 sensor is present. Trimming all data within EC = "+str(min_ec)+" time windows\n" + # find all time windows where EC is exactly 0 + ES2_data = data["EC_DECAGON"] + values = ES2_data["ec"].values + # print pose_lat_vals + # print pose_lon_vals + + # out_of_bouds_lat = (pose_lat_vals < data_boundaries[0][0]) | (pose_lat_vals > data_boundaries[1][0]) + # out_of_bouds_lon = (pose_lon_vals < data_boundaries[0][1]) | (pose_lon_vals > data_boundaries[1][1]) + + # ec_eq_zero_indices = np.where(values == 0)[0] + ec_eq_zero_indices = np.where( (values < min_ec) )[0] # | out_of_bouds_lat | out_of_bouds_lon )[0] + # ec_eq_zero_indices = np.where(values < 50)[0] + windows = list() + windows.append([ec_eq_zero_indices[0]]) + left = ec_eq_zero_indices[0] + for ii in range(1, ec_eq_zero_indices.shape[0]): + i = ec_eq_zero_indices[ii] + if i - left > 5: + # there has been a jump in index, a new time window has started + windows[-1].append(left) + windows.append([i]) + left = i + windows[-1].append(ec_eq_zero_indices[-1]) + # print ec_eq_zero_indices + # print windows + for window in windows: + time_window = [ES2_data["ec"].index.values[window[0]], ES2_data["ec"].index.values[window[1]]] + for k in data: + data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])] + else: + print "No ES2 sensor present. No trimming will be performed." + + + # print data + # if "PH_ATLAS" in data: + # print "pH sensor is present. Trimming all data within pH < 6 time windows\n" + # # find all time windows where pH is less than 6 + # pH_data = data["PH_ATLAS"] + # values = pH_data["ph"].values + # # pH_lt_6_indices = np.where( (values < 6) | (values > 8.5))[0] + # pH_lt_6_indices = np.where( (values < 6) )[0] + # windows = list() + # windows.append([pH_lt_6_indices[0]]) + # left = pH_lt_6_indices[0] + # for ii in range(1, pH_lt_6_indices.shape[0]): + # i = pH_lt_6_indices[ii] + # if i - left > 5: + # windows[-1].append(left) + # windows.append([i]) + # left = i + # windows[-1].append(pH_lt_6_indices[-1]) + # for window in windows: + # time_window = [pH_data["ph"].index.values[window[0]], pH_data["ph"].index.values[window[1]]] + # for k in data: + # data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])] + + # Print the available sensors and channels for this logfile. + print "Available sensors/channels:" + for s in data.keys(): + if s == 'pose' or s == 'BATTERY': + continue + for c in data[s].dtypes.keys(): + print " {:s}, {:s}".format(s, str(c)) + + + # Extract the pose timing and the sensor data of interest. + pose_times = pose.index.values.astype(np.float64) + + if sensor_name in data: + sensor = data[sensor_name] + sensor_times = sensor.index.values.astype(np.float64) + + # Linearly interpolate the position of the sensor at every sample. + sensor_pose_interpolator = scipy.interpolate.interp1d(pose_times, position, + axis=0, bounds_error=False) + + # Add the position information back to the sensor data. + sensor = sensor.join(pandas.DataFrame(sensor_pose_interpolator(sensor_times), sensor.index, + columns=('latitude', 'longitude'))) + + # Remove columns that have NaN values (no pose information). + sensor_valid = np.all(np.isfinite(sensor), axis=1) + sensor = sensor[sensor_valid] + + # Create a trail of the vehicle's path on the map. + pl = Polyline(locations=position.as_matrix().tolist()) + pl.fill_opacity = 0.0 + pl.weight = 2 + + ## Add a data overlay for the map + data_padding = [0.0000001, 0.0000001] # degrees lat/lon + data_resolution = [0.00001, 0.00001] # degrees lat/lon + data_interpolation_radius = 0.00001 # degrees lat/lon + data_bounds = [(position.min() - data_padding).tolist(), + (position.max() + data_padding).tolist()] + + print position.min() + print position.max() + print data_bounds + print data_resolution + + # Create a rectangular grid of overlay points. + data_xv, data_yv = np.meshgrid( + np.arange(data_bounds[1][0], data_bounds[0][0], -data_resolution[0]), + np.arange(data_bounds[0][1], data_bounds[1][1], data_resolution[1]) + ) + data_shape = data_xv.shape + data_xy = np.vstack([data_xv.ravel(), data_yv.ravel()]).T + + print data_shape + + print "starting major processing..." + if sensor_name in data: + # Create a radial-basis interpolator over the sensor dataset + # Then, query it at each point of the rectangular grid. + #from sklearn.neighbors import RadiusNeighborsClassifier + #data_estimator = RadiusNeighborsClassifier(radius=data_interpolation_radius, outlier_label=np.nan) + print "creating radius neighbors regressor" + data_estimator = RadiusNeighborsRegressor(radius=data_interpolation_radius) + + print "running neighbors fit" + data_estimator.fit(sensor[['latitude','longitude']], sensor[sensor_channel].astype(np.float)) + print "running predict" + data_zv = data_estimator.predict(data_xy) + print "running reshape" + data_zv = data_zv.reshape(data_shape).T + + print "normalizing data" + + # Normalize data from [0, 1) + data_max = data_zv[np.isfinite(data_zv)].max() + data_min = data_zv[np.isfinite(data_zv)].min() + print "Data min = {:f} Data max = {:f}".format(data_min, data_max) + NORMALIZER = data_max # 800 + data_zv = (data_zv - data_min) / (NORMALIZER - data_min) + + + print "create color map" + # Update a color map only at the points that have valid values. + data_rgb = np.zeros((data_shape[0], data_shape[1], 4), dtype=np.uint8) + data_rgb = matplotlib.cm.jet(data_zv) * 255 + data_rgb[:,:,3] = 255 * np.isfinite(data_zv) + + # Remove any old image files w/ the same name + old_png_files = glob.glob('./overlay/[overlay, tint, bar].png') + for old_png_file in old_png_files: + print "removing file: " + old_png_file + os.remove(old_png_file) + + out_prefix = log_file + "-"+sensor_name+"-"+str(min_ec)+'-' + + print "creating overlay files" + png_filename_rgb = './overlay/'+out_prefix+'overlay.png' + scipy.misc.imsave(png_filename_rgb, data_rgb) + + # Create image overlay that references generated image. + # makes an image w/ a slight opaque tint to it + # data_overlay_tint = np.ones((data_shape[0], data_shape[1], 4), dtype=np.uint8) * 50 + # data_overlay_tint_filename = './overlay/'+out_prefix+'tint.png' + # scipy.misc.imsave(data_overlay_tint_filename, data_overlay_tint) + + # data_bounds_tint = [(position.min() - [x * 500 for x in data_padding]).tolist(), + # (position.max() + [x * 500 for x in data_padding]).tolist()] + + # Make a figure and axes with dimensions as desired. + fig = pyplot.figure(figsize=(15, 3)) + ax1 = fig.add_axes([0.05, 0.80, 0.9, 0.15]) + # Set the colormap and norm to correspond to the data for which + # the colorbar will be used. + cmap = matplotlib.cm.jet + norm = matplotlib.colors.Normalize(vmin=data_min, vmax=NORMALIZER) + cb1 = matplotlib.colorbar.ColorbarBase(ax1, cmap=cmap, + norm=norm, + orientation='horizontal') + cb1.set_label(sensor_units) + + scale_bar_filename = './overlay/'+out_prefix+'bar.png' + pyplot.savefig(scale_bar_filename) + + pyplot.close('all') + +if __name__ == '__main__': + log_file = sys.argv[1] + sensor_id = int(sys.argv[2]) + min_ec = int(sys.argv[3]) + generate_overlay(log_file, sensor_id, min_ec) + # generate_histogram() diff --git a/image_server/image_server.py b/image_server/image_server.py index 9808f83..03ef098 100644 --- a/image_server/image_server.py +++ b/image_server/image_server.py @@ -22,6 +22,13 @@ app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER +class PlatypusDataStore(object): + """docstring for PlatypusDataStore""" + def __init__(self, data): + super(PlatypusDataStore, self).__init__() + self.data = data + + @app.route('/path/') def send(path): if (os.path.exists(path)): @@ -30,8 +37,30 @@ def send(path): else: print path +" does not exist" -@app.route('/logs/') -def show_log(log_file): +def sensor_id_to_name(id): + if (id == 0): + # Select the sensor and the name of the channel for that sensor. + sensor_name = 'PH_ATLAS' + sensor_channel = 'ph' + sensor_units = "pH" + elif (id == 1): + sensor_name = 'EC_DECAGON' + sensor_channel = 'ec' + sensor_units = 'Electrical Conductivity (uS/cm)' + elif (id == 2): + sensor_name = 'T_DECAGON' + sensor_channel = 'temperature' + sensor_units = 'Temperature (C)' + elif (id == 3): + sensor_name = 'DO_ATLAS' + sensor_channel = 'do' + sensor_units = 'Turbidty (NTU)' + + return (sensor_name, sensor_channel, sensor_units) + + +@app.route('/logs////overlay.jpg') +def generate_overlay(log_file, sensor_id, min_ec): # Import the data from the specified logfile log_ext = ".txt" @@ -41,10 +70,10 @@ def show_log(log_file): log_path + log_file ] - csv_output_filename = "ERM_2018_SF" data = platypus.io.logs.merge_files(log_filenames) + # data_store = PlatypusDataStore(data) if "EC_DECAGON" in data: print "ES2 sensor is present. Trimming all data within EC = 0 time windows\n" @@ -52,7 +81,7 @@ def show_log(log_file): ES2_data = data["EC_DECAGON"] values = ES2_data["ec"].values # ec_eq_zero_indices = np.where(values == 0)[0] - ec_eq_zero_indices = np.where( (values < 2000))[0] + ec_eq_zero_indices = np.where( (values < min_ec))[0] # ec_eq_zero_indices = np.where(values < 50)[0] windows = list() windows.append([ec_eq_zero_indices[0]]) @@ -74,8 +103,6 @@ def show_log(log_file): else: print "No ES2 sensor present. No trimming will be performed." - - # if "PH_ATLAS" in data: # print "pH sensor is present. Trimming all data within pH < 6 time windows\n" # # find all time windows where pH is less than 6 @@ -111,22 +138,6 @@ def show_log(log_file): for c in data[s].dtypes.keys(): print " {:s}, {:s}".format(s, str(c)) - # Select the sensor and the name of the channel for that sensor. - sensor_name = 'PH_ATLAS' - sensor_channel = 'ph' - sensor_units = "pH" - - sensor_name = 'EC_DECAGON' - sensor_channel = 'ec' - sensor_units = 'Electrical Conductivity (uS/cm)' - - # sensor_name = 'T_DECAGON' - # sensor_channel = 'temperature' - # sensor_units = 'Temperature (C)' - - sensor_name = 'DO_ATLAS' - sensor_channel = 'do' - sensor_units = 'Turbidty (NTU)' # Extract the pose timing and the sensor data of interest. pose_times = pose.index.values.astype(np.float64) @@ -143,14 +154,11 @@ def show_log(log_file): sensor = sensor.join(pandas.DataFrame(sensor_pose_interpolator(sensor_times), sensor.index, columns=('latitude', 'longitude'))) - # print sensor data to csv file - sensor.to_csv(log_path + csv_output_filename + "__" + sensor_name + ".csv") - # Remove columns that have NaN values (no pose information). sensor_valid = np.all(np.isfinite(sensor), axis=1) sensor = sensor[sensor_valid] - # Create a trail of the vehicle's path on the map. + # Create a trail of the vehicle's path on the map. pl = Polyline(locations=position.as_matrix().tolist()) pl.fill_opacity = 0.0 pl.weight = 2 diff --git a/image_server/templates/index.html b/image_server/templates/index.html index 2d48c32..7c1f2e8 100644 --- a/image_server/templates/index.html +++ b/image_server/templates/index.html @@ -11,9 +11,6 @@ - - - @@ -22,6 +19,8 @@

{{ log_filenames }}!

+ + - - - diff --git a/notebooks/Data_Interpolation.ipynb b/notebooks/Data_Interpolation.ipynb index 97c995e..6439b03 100644 --- a/notebooks/Data_Interpolation.ipynb +++ b/notebooks/Data_Interpolation.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 56, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -25,7 +25,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 2, "metadata": { "scrolled": false }, @@ -34,18 +34,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "here: DO_ATLAS\n", "here: BATTERY\n", - "here: EC_DECAGON\n", "here: PH_ATLAS\n", - "here: T_DECAGON\n", - "ES2 sensor is present. Trimming all data within EC = 0 time windows\n", - "\n", + "here: DO_ATLAS\n", + "No ES2 sensor present. No trimming will be performed.\n", "Available sensors/channels:\n", - " DO_ATLAS, do\n", - " EC_DECAGON, ec\n", " PH_ATLAS, ph\n", - " T_DECAGON, temperature\n" + " DO_ATLAS, do\n" ] } ], @@ -71,13 +66,13 @@ " log_path + \"platypus_20180604_105536.txt\"\n", "]\n", "\n", - "# log_path = \"/home/shawn/Downloads/data/day1\"\n", - "# log_filenames = glob.glob(log_path+'/'+\"*.txt\")\n", + "log_path = \"/home/shawn/data/june 18 2018 - NL delfgauw/day_2\"\n", + "log_filenames = glob.glob(log_path+'/'+\"*.txt\")\n", "\n", "# log_path = \"/home/shawn/NL1/all_nov_2017/\"\n", "# log_filenames = glob.glob(log_path+'/'+\"*.txt\")\n", "\n", - "csv_output_filename = \"ERM_2018_SF\"\n", + "csv_output_filename = \"NL_Delfgauw_June_2018_SF\"\n", "\n", "# data = platypus.io.logs.load(log_path + log_filename + log_ext)\n", "data = platypus.io.logs.merge_files(log_filenames)\n", @@ -151,7 +146,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -160,22 +155,24 @@ "sensor_channel = 'ph'\n", "sensor_units = \"pH\"\n", "\n", - "sensor_name = 'EC_DECAGON'\n", - "sensor_channel = 'ec'\n", - "sensor_units = 'Electrical Conductivity (uS/cm)'\n", + "# sensor_name = 'EC_DECAGON'\n", + "# sensor_channel = 'ec'\n", + "# sensor_units = 'Electrical Conductivity (uS/cm)'\n", "\n", "# sensor_name = 'T_DECAGON'\n", "# sensor_channel = 'temperature'\n", "# sensor_units = 'Temperature (C)'\n", "\n", - "sensor_name = 'DO_ATLAS'\n", - "sensor_channel = 'do'\n", - "sensor_units = 'Turbidty (NTU)'" + "# sensor_name = 'DO_ATLAS'\n", + "# sensor_channel = 'do'\n", + "# sensor_units = 'Turbidty (NTU)'\n", + "# print data[\"pose\"]\n", + "data[\"pose\"].to_csv(log_path + \"/poses.csv\", columns=[\"latitude\", \"longitude\"])" ] }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 11, "metadata": { "scrolled": true }, From c54f85bb0e543dc5e1ee8049ca321e648cad3e50 Mon Sep 17 00:00:00 2001 From: Shawn Hanna Date: Thu, 28 Jun 2018 16:24:31 -0400 Subject: [PATCH 30/43] starting work on processor --- image_server/data_processor.py | 2 - notebooks/Data_Interpolation.ipynb | 86 +++++++++++++++++++----------- src/platypus/io/insitu_logs.py | 59 ++++++++++++++++++++ src/platypus/io/logs.py | 2 +- 4 files changed, 116 insertions(+), 33 deletions(-) create mode 100644 src/platypus/io/insitu_logs.py diff --git a/image_server/data_processor.py b/image_server/data_processor.py index 05a3390..7d75c61 100644 --- a/image_server/data_processor.py +++ b/image_server/data_processor.py @@ -54,8 +54,6 @@ def generate_overlay(log_file, sensor_id, min_ec): log_path + log_file + log_ext ] - - data = platypus.io.logs.merge_files(log_filenames) (sensor_name, sensor_channel, sensor_units) = sensor_id_to_name(sensor_id) diff --git a/notebooks/Data_Interpolation.ipynb b/notebooks/Data_Interpolation.ipynb index 6439b03..cc23f2f 100644 --- a/notebooks/Data_Interpolation.ipynb +++ b/notebooks/Data_Interpolation.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -25,7 +25,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 29, "metadata": { "scrolled": false }, @@ -34,13 +34,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "here: BATTERY\n", - "here: PH_ATLAS\n", - "here: DO_ATLAS\n", - "No ES2 sensor present. No trimming will be performed.\n", + "['/home/shawn/Downloads/Archive/platypus_20180221_172549.txt', '/home/shawn/Downloads/Archive/platypus_20180619_112857.txt', '/home/shawn/Downloads/Archive/platypus_20180618_130514.txt', '/home/shawn/Downloads/Archive/platypus_20180619_093422.txt', '/home/shawn/Downloads/Archive/platypus_20180617_214447.txt', '/home/shawn/Downloads/Archive/platypus_20180617_200121.txt']\n", + "set([u'DO_ATLAS', u'BATTERY', 'pose', u'EC_DECAGON', u'PH_ATLAS', u'T_DECAGON'])\n", + "ES2 sensor is present. Trimming all data within EC = 0 time windows\n", + "\n", "Available sensors/channels:\n", + " DO_ATLAS, do\n", + " EC_DECAGON, ec\n", " PH_ATLAS, ph\n", - " DO_ATLAS, do\n" + " T_DECAGON, temperature\n" ] } ], @@ -49,34 +51,52 @@ "\n", "log_ext = \".txt\"\n", "\n", - "log_path = \"/home/shawn/data/ERM/log_files/\"\n", + "log_path = \"/home/shawn/Downloads/Archive\"\n", "log_filenames = [\n", - "# log_path + \"platypus_20180213_022417.txt\",\n", - "# log_path + \"platypus_20180213_030409.txt\",\n", - "# log_path + \"platypus_20180213_035212.txt\",\n", - "# log_path + \"platypus_20180213_042357.txt\",\n", - "# log_path + \"platypus_20180213_054659.txt\",\n", - "# log_path + \"platypus_20180213_060554.txt\",\n", - "# log_path + \"platypus_20180213_062547.txt\",\n", - "# log_path + \"platypus_20180213_065743.txt\",\n", - "# log_path + \"platypus_20180213_073112.txt\",\n", - "# log_path + \"platypus_20180213_074623.txt\",\n", - "# log_path + \"platypus_20180213_082044.txt\",\n", - "# log_path + \"platypus_20180213_085649.txt\",\n", - " log_path + \"platypus_20180604_105536.txt\"\n", + "# log_path + \"/\" + \"platypus_20180619_120536.txt\",\n", + "# log_path + \"/\" + \"platypus_20180619_120706.txt\",\n", + "# log_path + \"/\" + \"platypus_20180619_130313.txt\",\n", + "# log_path + \"/\" + \"platypus_20180619_131024.txt\",\n", + "# log_path + \"/\" + \"platypus_20180619_133227.txt\",\n", + "# log_path + \"/\" + \"platypus_20180619_133645.txt\",\n", + "# log_path + \"/\" + \"platypus_20180619_141055.txt\",\n", + "# log_path + \"/\" + \"platypus_20180619_141145.txt\",\n", + "# log_path + \"/\" + \"platypus_20180619_141345.txt\",\n", + "# log_path + \"/\" + \"platypus_20180619_141356.txt\",\n", + "# log_path + \"/\" + \"platypus_20180619_141442.txt\",\n", + "# log_path + \"/\" + \"platypus_20180619_161419.txt\",\n", + "# log_path + \"/\" + \"platypus_20180619_161511.txt\",\n", + "# log_path + \"/\" + \"platypus_20180619_165615.txt\",\n", + "# log_path + \"/\" + \"platypus_20180620_092526.txt\",\n", + "# log_path + \"/\" + \"platypus_20180620_120001.txt\",\n", + "# log_path + \"/\" + \"platypus_20180620_123406.txt\",\n", + "# log_path + \"/\" + \"platypus_20180620_134304.txt\",\n", + "# log_path + \"/\" + \"platypus_20180620_161350.txt\",\n", + "# log_path + \"/\" + \"platypus_20180620_164312.txt\",\n", + "# log_path + \"/\" + \"platypus_20180620_173053.txt\",\n", + "# log_path + \"/\" + \"platypus_20180620_173712.txt\",\n", + "# log_path + \"/\" + \"platypus_20180620_175217.txt\",\n", + "# log_path + \"/\" + \"platypus_20180620_181927.txt\",\n", + "# log_path + \"/\" + \"platypus_20180620_182042.txt\",\n", + "\n", "]\n", "\n", - "log_path = \"/home/shawn/data/june 18 2018 - NL delfgauw/day_2\"\n", + "log_path = \"/home/shawn/Downloads/Archive/\"\n", "log_filenames = glob.glob(log_path+'/'+\"*.txt\")\n", + "print log_filenames\n", "\n", "# log_path = \"/home/shawn/NL1/all_nov_2017/\"\n", "# log_filenames = glob.glob(log_path+'/'+\"*.txt\")\n", "\n", - "csv_output_filename = \"NL_Delfgauw_June_2018_SF\"\n", + "csv_output_filename = \"NL_Nootdorp_June_2018\"\n", "\n", "# data = platypus.io.logs.load(log_path + log_filename + log_ext)\n", "data = platypus.io.logs.merge_files(log_filenames)\n", "\n", + "for x in log_filenames:\n", + " if not os.path.exists:\n", + " print \"file \" + x + \" does not exist!!!\"\n", + "\n", "\n", "if \"EC_DECAGON\" in data:\n", " print \"ES2 sensor is present. Trimming all data within EC = 0 time windows\\n\"\n", @@ -84,7 +104,7 @@ " ES2_data = data[\"EC_DECAGON\"]\n", " values = ES2_data[\"ec\"].values\n", "# ec_eq_zero_indices = np.where(values == 0)[0]\n", - " ec_eq_zero_indices = np.where( (values < 2000))[0]\n", + " ec_eq_zero_indices = np.where( (values < 20))[0]\n", "# ec_eq_zero_indices = np.where(values < 50)[0]\n", " windows = list()\n", " windows.append([ec_eq_zero_indices[0]])\n", @@ -146,7 +166,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -165,14 +185,20 @@ "\n", "# sensor_name = 'DO_ATLAS'\n", "# sensor_channel = 'do'\n", - "# sensor_units = 'Turbidty (NTU)'\n", + "# sensor_units = 'Dissolved Oxygen'\n", + "\n", + "\n", "# print data[\"pose\"]\n", - "data[\"pose\"].to_csv(log_path + \"/poses.csv\", columns=[\"latitude\", \"longitude\"])" + "# data[\"pose\"].to_csv(log_path + \"/poses.csv\", columns=[\"latitude\", \"longitude\"])\n", + "data[\"PH_ATLAS\"][\"ph\"].to_csv(log_path + \"/ph.csv\")\n", + "data[\"DO_ATLAS\"][\"do\"].to_csv(log_path + \"/do.csv\")\n", + "data[\"EC_DECAGON\"][\"ec\"].to_csv(log_path + \"/ec.csv\")\n", + "data[\"T_DECAGON\"][\"temperature\"].to_csv(log_path + \"/temp.csv\")" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 37, "metadata": { "scrolled": true }, @@ -203,14 +229,14 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "/home/shawn/Downloads/analytics-shawn_netherlands_save/analytics-shawn_netherlands_save/venv/lib/python2.7/site-packages/ipykernel_launcher.py:2: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n", + "/home/shawn/src/analytics/venv/lib/python2.7/site-packages/ipykernel_launcher.py:2: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n", " \n" ] } diff --git a/src/platypus/io/insitu_logs.py b/src/platypus/io/insitu_logs.py new file mode 100644 index 0000000..7d6459f --- /dev/null +++ b/src/platypus/io/insitu_logs.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +""" +Module for handling the import of various logfiles into numpy arrays. +Copyright 2015. Platypus LLC. All rights reserved. +""" +import collections +import datetime +import logging +import itertools +import pandas + +logger = logging.getLogger(__name__) + + +def merge_files(filename_list): + """ + + :param: filename_list: list of full path filename strings + :return: One result will all the dataframes merged + :rtype: {str: pandas.DataFrame} + """ + + logfile_result_list = [load(filename) for filename in filename_list] + if len(logfile_result_list) == 1: + return logfile_result_list[0] + #all_data_types = set() + #for i in range(1, len(logfile_result_list)): + # all_data_types = all_data_types.union(set(logfile_result_list[i].keys())) + all_data_types = {key for log_dict in logfile_result_list for key in log_dict.keys()} + print all_data_types + + merged_dataframe_dict = dict() + + for data_type in all_data_types: + for i in range(len(logfile_result_list)): + if data_type in logfile_result_list[i]: + first_log_index = i + break + merged_dataframe_dict[data_type] = logfile_result_list[first_log_index][data_type] + for i in range(first_log_index + 1, len(logfile_result_list)): + if data_type in logfile_result_list[i]: + merged_dataframe_dict[data_type] = merged_dataframe_dict[data_type].combine_first(logfile_result_list[i][data_type]).dropna(how='any') + return merged_dataframe_dict + + + + +def load(filename): + """ + Loads a log from an in-situ log file (htm or csv) + + Attempts to auto-detect format from the file. + + :param filename: path to a log file + :type filename: string + :returns: a dict containing the data from this logfile + :rtype: {str: numpy.recarray} + """ + return pandas.read_csv(filename) diff --git a/src/platypus/io/logs.py b/src/platypus/io/logs.py index 6d49f8f..09dca13 100644 --- a/src/platypus/io/logs.py +++ b/src/platypus/io/logs.py @@ -577,7 +577,7 @@ def read(logfile, filename=None): # Depending on the format of the first line, pick an appropriate loader. if len(components[1]) == 1: # Version 4.2.0 files have a single-character log-level. - return read_v4_2_0(logfile) + return read_v4_3_0(logfile) else: try: # Version 4.1.0 logs have JSON messages. From e47f474cdeace1150fcdd24554a4175bebe6b403 Mon Sep 17 00:00:00 2001 From: Shawn Hanna Date: Thu, 28 Jun 2018 16:26:20 -0400 Subject: [PATCH 31/43] added pose fixer --- examples/pose_fixer.py | 78 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 examples/pose_fixer.py diff --git a/examples/pose_fixer.py b/examples/pose_fixer.py new file mode 100644 index 0000000..3172c36 --- /dev/null +++ b/examples/pose_fixer.py @@ -0,0 +1,78 @@ +import sys +import csv +from csv import DictWriter, DictReader +from dateutil import parser +import datetime +import os +import glob +import append_csvs + +def get_poses(filename_poses, pose_offset): + poses = [] + with open(filename_poses) as csvfile: + reader = DictReader(csvfile) + print "pose offset: " + str(pose_offset) + last_row = {"time": 0, "latitude": 0, "longitude": 0} + for row in reader: + dt = parser.parse(row["time"]) + datetime.timedelta(hours = pose_offset) + if (last_row["latitude"] == row["latitude"] and last_row["longitude"] == row["longitude"] and abs(last_row["time"] - dt) < datetime.timedelta(seconds = 1)): + continue + row["time"] = dt + poses.append(row) + last_row = row + + print "first point: " + str(poses[0]) +"\nLast point: "+ str(poses[-1])+"\nnum points: "+str(len(poses))+"\n\n" + return poses + +def fix_insitu_csv(filename_insitu, poses, output_filename): + with open(filename_insitu) as csvfile: + reader = DictReader(csvfile) + + with open(output_filename, 'w') as csvoutfile: + fieldnames = reader.fieldnames + writer = csv.DictWriter(csvoutfile, fieldnames=fieldnames) + writer.writeheader() + print "insitu data: " + for row in reader: + # print(row) + insitu_date = parser.parse(row['Date Time']) + latest_time_diff = 0 + min_val = min(poses, key=lambda x:abs(x["time"]-insitu_date)) + # for x in poses: + # pose_date = x["time"] + # if (pose_date > insitu_date): + # print "found a pose after " + str(pose_date) + " for insitu data point: " + str(insitu_date) + " - diff = " + str(abs(insitu_date -pose_date)) + diff_time = abs(insitu_date - min_val["time"]) + if (diff_time > datetime.timedelta(seconds = 1)): + print "found the closest pose (" + str(min_val) + ") for insitu data point: " + str(insitu_date) + " - diff = " + str(diff_time) + + row['Date Time'] = min_val["time"] + row['Latitude'] = min_val["latitude"] + row['Longitude'] = min_val["longitude"] + writer.writerow(row) + +if __name__ == '__main__': + filename_poses = sys.argv[1] + pose_offset = int(sys.argv[2]) + filename_insitu = sys.argv[3] + if (os.path.exists(filename_insitu) == False): + print("file doesn't exist: "+filename_insitu) + + "filename of poses: ", filename_poses + dict_poses = get_poses(filename_poses, pose_offset) + + if (os.path.isdir(filename_insitu)): + files_in_folder = glob.glob(filename_insitu+'/VuSitu_*.csv') + else: + files_in_folder = [filename_insitu] + + + files_output = [] + + for x in files_in_folder: + print "filename of insitu data: ", x + fix_insitu_csv(x, dict_poses, x+".fixed") + files_output.append(x + ".fixed") + + if os.path.isdir(filename_insitu): + append_csvs(files = files_output, output_filename = filename_insitu + "/combined.csv") \ No newline at end of file From 3934e158343addae566b2c9fb2a363bbef47cef4 Mon Sep 17 00:00:00 2001 From: Shawn Hanna Date: Tue, 10 Jul 2018 09:08:00 -0400 Subject: [PATCH 32/43] added histogram for insitu data --- examples/histogram-insitu.py | 162 +++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 examples/histogram-insitu.py diff --git a/examples/histogram-insitu.py b/examples/histogram-insitu.py new file mode 100644 index 0000000..fd90158 --- /dev/null +++ b/examples/histogram-insitu.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python +# coding: utf-8 + +""" +Example of loading Platypus vehicle logs from file. + +Data is loaded as time series into Pandas[1] DataFrames, from which they can +be interpolated and filtered for other purposes, or used as input into Numpy[2] +or Scipy[3] functions. + +[1]: http://pandas.pydata.org/ +[2]: http://www.numpy.org/ +[3]: http://www.scipy.org/ +""" +import matplotlib.pyplot as plt +import platypus.io.insitu_logs +import platypus.util.conversions +import glob +import os +import numpy as np +import math + +# Read the data log from file. +# Note: for log versions <5.0, this filename must be 'airboat_[timestamp].txt]. + +def trim_using_EC(dataframe, threshold=100): + """ + Trims any data when EC < 100 + :return: trimmed dataframe + """ + if "ES2" in dataframe: + print("ES2 sensor is present. Trimming all data within EC < {:.0f} time windows\n".format(threshold)) + # find all time windows where EC is exactly 0 + ES2_data = dataframe["ES2"] + values = ES2_data["ec"].values + ec_eq_zero_indices = np.where(values < threshold)[0] + windows = list() + windows.append([ec_eq_zero_indices[0]]) + left = ec_eq_zero_indices[0] + for ii in range(1, ec_eq_zero_indices.shape[0]): + i = ec_eq_zero_indices[ii] + if i - left > 5: + # there has been a jump in index, a new time window has started + windows[-1].append(left) + windows.append([i]) + left = i + windows[-1].append(ec_eq_zero_indices[-1]) + # print(ec_eq_zero_indices) + # print(windows) + for window in windows: + time_window = [ES2_data["ec"].index.values[window[0]], ES2_data["ec"].index.values[window[1]]] + for k in dataframe: + dataframe[k] = dataframe[k].loc[np.logical_or(dataframe[k].index < time_window[0], dataframe[k].index > time_window[1])] + else: + print("No ES2 sensor present. No trimming will be performed.") + return dataframe + + +def load_data(folders = [], files = [], ec_trim_value = 50): + if (len(folders) == 0): + folders = [sys.path.cwd] + + # print("folders", str(folders)) + + for folder in folders: + files.extend(glob.glob(folder+'/*fixed*.csv')) + + # print("log files: " + str(files)) + # todo: remove duplicates? + + data = platypus.io.insitu_logs.merge_files(files) + data = trim_using_EC(data, ec_trim_value) + return data + +def plot_hist_sensor(data, sensor = 'ES2', num_bins = 10, hide_top_n_percent = 0, hide_bot_n_percent = 0, save_dir = "~/save"): + num_readings = len(data[sensor]) + # Get the std of the data + sensor_stddev = data[sensor].std() + # Get the mean of the data + sensor_mean = data[sensor].mean() + # Get the min of the data + sensor_min = data[sensor].min() + # Get the max of the data + sensor_max = data[sensor].max() + + print(sensor+" number of readings", num_readings) + print(sensor+" std", sensor_stddev) + print(sensor+" mean", sensor_mean) + print(sensor+" min", sensor_min) + print(sensor+" max", sensor_max) + + hist_max = math.ceil(sensor_max - hide_top_n_percent * 0.01 * sensor_max) + hist_min = math.floor(sensor_min + hide_bot_n_percent * 0.01 * sensor_min) + bin_size = (hist_max - hist_min)/float(num_bins) + + bins = np.arange(hist_min, hist_max, bin_size) + # print(hist_max, hist_min, bin_size, bins) + + # n, bins, patches = plt.hist(data[sensor], bins=xrange(200,1600,100)) + weights = np.ones_like(data[sensor])/float(num_readings) * 100 + if (num_bins <= 0): + n, bins, patches = plt.hist(data[sensor], weights=weights) + else: + n, bins, patches = plt.hist(data[sensor], weights=weights, bins=bins) + + # print(n, bins, patches) + + plt.xlabel(sensor) + plt.ylabel('Percentage of values in the given range') + plt.ylim(0,50) + plt.title('Histogram of ' + sensor + ' ' + save_dir.split('/')[-1]) + plt.savefig(save_dir + "/"+'Histogram of ' + sensor + ' ' + save_dir.split('/')[-1]+'.png') + # plt.text(0, .25, "Standard Dev: " + str(es2_stddev)) + plt.figtext(.16, .75, "Mean: " + str(sensor_mean)) + plt.figtext(.16, .7, "std: " + str(sensor_stddev)) + plt.grid(True) + plt.show() + +def get_folders(): + # folders = ['/home/shawn/NL2/grokeneveldse_polder/grokeneveldse_polder_feb_2018/'] + folders = ['/home/shawn/data/june 18 2018 - NL delfgauw/day_1', '/home/shawn/data/june 18 2018 - NL delfgauw/day_2'] + return folders + +def main(): + print("enter EC trim value: ") + new_ec_trim = int(raw_input()) + + folders = get_folders() + + data = load_data(folders=folders, ec_trim_value = new_ec_trim) + print(data) + print("data columns: ", data.keys) + + num_bins = 10 + hide_bot_n_percent = 10 + hide_top_n_percent = 10 + while ( True ): + print("what would you like to do?") + print("0: "+"change number of bins (currently: " +str(num_bins)+")") + print("1: "+"change percentage of bottom to hide (currently: " +str(hide_bot_n_percent)+")") + print("2: "+"change percentage of top to hide (currently: " +str(hide_top_n_percent)+")") + for i,x in enumerate(data.keys()): + print(i+3, ": plot " + x) + + command = raw_input() + + if (command == '0'): + break + elif (command == '1'): + num_bins = int(command) + elif (command == '2'): + hide_bot_n_percent = int(command) + elif (command == '3'): + hide_top_n_percent = int(command) + elif(int(command) < len(data.keys()) ): + plot_hist_sensor(data, data.keys()[int(command)], num_bins = num_bins, hide_top_n_percent = hide_top_n_percent, hide_bot_n_percent=hide_bot_n_percent, save_dir = folders[0]) + else: + print(command +" is not valid") + + +if __name__ == '__main__': + main() From e27ecca3013002fde82862bfe9d1e61ee5060fb2 Mon Sep 17 00:00:00 2001 From: Shawn Hanna Date: Tue, 10 Jul 2018 15:22:52 -0400 Subject: [PATCH 33/43] added more functionality to data processor and image server. seems to work for a specific use case --- image_server/data_processor.py | 300 +++++++++++++++++++---------- image_server/image_server.py | 254 +++++------------------- image_server/index.html | 8 - image_server/templates/index.html | 48 +---- image_server/templates/render.html | 56 ++++++ 5 files changed, 304 insertions(+), 362 deletions(-) delete mode 100644 image_server/index.html create mode 100644 image_server/templates/render.html diff --git a/image_server/data_processor.py b/image_server/data_processor.py index 7d75c61..7076bd8 100644 --- a/image_server/data_processor.py +++ b/image_server/data_processor.py @@ -1,5 +1,3 @@ -from ipyleaflet import Map, ImageOverlay, Polyline - from sklearn.neighbors import RadiusNeighborsRegressor import matplotlib import matplotlib.cm @@ -13,6 +11,7 @@ import os import glob import sys +import json def sensor_id_to_name(id): if (id == 0): @@ -35,13 +34,29 @@ def sensor_id_to_name(id): return (sensor_name, sensor_channel, sensor_units) -def generate_overlay(log_file, sensor_id, min_ec): +def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds): + # read the old generation stats file + stat_in = {} + + try: + with open("./stats/"+log_file+'.json', 'r') as infile: + print "reading previous stats from: " + "./stats/"+log_file+'.json' + stats_in = json.load(infile) + except: + print "failed to load from input stats file" + + data_stats = {} + data_stats["settings"] = {} + data_stats["settings"]["log_path"] = log_path + data_stats["settings"]["log_file"] = log_file + data_stats["settings"]["sensor_id"] = sensor_id + data_stats["settings"]["ec_bounds"] = ec_bounds + data_stats["settings"]["ph_bounds"] = ph_bounds + # Import the data from the specified logfile log_ext = ".txt" - log_path = "/home/shawn/data/ERM/log_files/" - if (os.path.exists(log_path + log_file + log_ext) == False): print "File doesn't exist: " + log_path + log_file + log_ext log_ext = ".txt.incomplete" @@ -60,85 +75,120 @@ def generate_overlay(log_file, sensor_id, min_ec): # Define useful access variables. pose = data['pose'] - position = pose[['latitude', 'longitude']] - - data_boundaries = [ [37.755690, -122.381139], [37.757928, -122.380338]] - if "T_DECAGON" in data: - print "Temperature sensor is present. Trimming all data where temperature is changing a lot in a given time windows\n" + data_boundaries = [ [37.751816, -122.384368], [37.764696, -122.366599]] + if (data_boundaries != []): + print "Trimming all data within long/lat = "+str(data_boundaries) # find all time windows where EC is exactly 0 - - # print ES2_data - T_data = data["T_DECAGON"] - values = T_data["temperature"].values - - dtemp_dt_limit = 0.5/60.0 - - # pose_lat_vals = position["latitude"].values - # pose_lon_vals = position["longitude"].values - stddevs = [] - zero_indices = [] - for i, x in enumerate(values): - zero_indices.append(0) - stddevs.append(0) - if (i + 60 < len(values)): - vals = [] - for x in xrange(i, i+60): - vals.append(values[x]) - vals = np.array(vals) - stddev = vals.std() - stddevs[i] = stddev - # if (stddev > 0.1): - # print "@ i = " + str(i)+", time = " + str(T_data["temperature"].index[i]) + " - std dev: " + str(stddev) - - for i, x in enumerate(stddevs): - # print i - if (x > 0.1): - # print x - for y in xrange(i, i+60): - zero_indices[y] = 1 - - # print zero_indices + ES2_data = data["pose"] + # print ES2_data["time"] + values_lat = ES2_data["latitude"].values + values_lon = ES2_data["longitude"].values # ec_eq_zero_indices = np.where(values == 0)[0] - ec_eq_zero_indices = np.where( (np.array(zero_indices) == 1) )[0] # | out_of_bouds_lat | out_of_bouds_lon )[0] - # print ec_eq_zero_indices - # ec_eq_zero_indices = np.where(values < 50)[0] - windows = list() - windows.append([ec_eq_zero_indices[0]]) - left = ec_eq_zero_indices[0] - for ii in range(1, ec_eq_zero_indices.shape[0]): - i = ec_eq_zero_indices[ii] - if i - left > 5: - # there has been a jump in index, a new time window has started - windows[-1].append(left) - windows.append([i]) - left = i - windows[-1].append(ec_eq_zero_indices[-1]) - # print ec_eq_zero_indices - # print windows - for window in windows: - print "window: " + str(window) - time_window = [T_data["temperature"].index.values[window[0]], T_data["temperature"].index.values[window[1]]] - for k in data: - data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])] - print T_data["temperature"].values[np.where( (np.array(zero_indices) ==1) )[0]] - else: - print "No ES2 sensor present. No trimming will be performed." + ec_eq_zero_indices = np.where( (values_lat < data_boundaries[0][0]) | (values_lat > data_boundaries[1][0]) | + (values_lon < data_boundaries[0][1]) | (values_lon > data_boundaries[1][1]) )[0] + + if (len(ec_eq_zero_indices) == 0): + print "no poses outside the bounds" + else: + # print ec_eq_zero_indices + # ec_eq_zero_indices = np.where(values < 50)[0] + windows = list() + windows.append([ec_eq_zero_indices[0]]) + left = ec_eq_zero_indices[0] + for ii in range(1, ec_eq_zero_indices.shape[0]): + i = ec_eq_zero_indices[ii] + if i - left > 5: + # there has been a jump in index, a new time window has started + windows[-1].append(left) + windows.append([i]) + left = i + windows[-1].append(ec_eq_zero_indices[-1]) + # print ec_eq_zero_indices + # print windows + for window in windows: + time_window = [ES2_data.index.values[window[0]], ES2_data.index.values[window[1]]] + # print time_window + for k in data: + print "trimming: " + k +" for interval " + str(time_window) + data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])] + + + # if "T_DECAGON" in data: + # print "Temperature sensor is present. Trimming all data where temperature is changing a lot in a given time windows\n" + # # find all time windows where EC is exactly 0 + + # # print ES2_data + # T_data = data["T_DECAGON"] + # values = T_data["temperature"].values + + # dtemp_dt_limit = 0.5/60.0 + + # # pose_lat_vals = position["latitude"].values + # # pose_lon_vals = position["longitude"].values + # stddevs = [] + # zero_indices = [] + # for i, x in enumerate(values): + # zero_indices.append(0) + # stddevs.append(0) + # if (i + 60 < len(values)): + # vals = [] + # for x in xrange(i, i+60): + # vals.append(values[x]) + # vals = np.array(vals) + # stddev = vals.std() + # stddevs[i] = stddev + # # if (stddev > 0.1): + # # print "@ i = " + str(i)+", time = " + str(T_data["temperature"].index[i]) + " - std dev: " + str(stddev) + + # for i, x in enumerate(stddevs): + # # print i + # if (x > 0.1): + # # print x + # for y in xrange(i, i+60): + # zero_indices[y] = 1 + + # # # print zero_indices + + # # ec_eq_zero_indices = np.where(values == 0)[0] + # ec_eq_zero_indices = np.where( (np.array(zero_indices) == 1) )[0] # | out_of_bouds_lat | out_of_bouds_lon )[0] + # # print ec_eq_zero_indices + # # ec_eq_zero_indices = np.where(values < 50)[0] + # windows = list() + # windows.append([ec_eq_zero_indices[0]]) + # left = ec_eq_zero_indices[0] + # for ii in range(1, ec_eq_zero_indices.shape[0]): + # i = ec_eq_zero_indices[ii] + # if i - left > 5: + # # there has been a jump in index, a new time window has started + # windows[-1].append(left) + # windows.append([i]) + # left = i + # windows[-1].append(ec_eq_zero_indices[-1]) + # # print ec_eq_zero_indices + # # print windows + # for window in windows: + # print "window: " + str(window) + # time_window = [T_data["temperature"].index.values[window[0]], T_data["temperature"].index.values[window[1]]] + # for k in data: + # data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])] + # print T_data["temperature"].values[np.where( (np.array(zero_indices) ==1) )[0]] + # else: + # print "No ES2 sensor present. No trimming will be performed." + + + min_ec = ec_bounds[0] + max_ec = ec_bounds[1] if "EC_DECAGON" in data: print "ES2 sensor is present. Trimming all data within EC = "+str(min_ec)+" time windows\n" # find all time windows where EC is exactly 0 ES2_data = data["EC_DECAGON"] values = ES2_data["ec"].values - # print pose_lat_vals - # print pose_lon_vals - - # out_of_bouds_lat = (pose_lat_vals < data_boundaries[0][0]) | (pose_lat_vals > data_boundaries[1][0]) - # out_of_bouds_lon = (pose_lon_vals < data_boundaries[0][1]) | (pose_lon_vals > data_boundaries[1][1]) # ec_eq_zero_indices = np.where(values == 0)[0] - ec_eq_zero_indices = np.where( (values < min_ec) )[0] # | out_of_bouds_lat | out_of_bouds_lon )[0] + ec_eq_zero_indices = np.where( (values < min_ec) | (values > max_ec) )[0] # | out_of_bouds_lat | out_of_bouds_lon )[0] # ec_eq_zero_indices = np.where(values < 50)[0] windows = list() windows.append([ec_eq_zero_indices[0]]) @@ -160,29 +210,38 @@ def generate_overlay(log_file, sensor_id, min_ec): else: print "No ES2 sensor present. No trimming will be performed." - - # print data - # if "PH_ATLAS" in data: - # print "pH sensor is present. Trimming all data within pH < 6 time windows\n" - # # find all time windows where pH is less than 6 - # pH_data = data["PH_ATLAS"] - # values = pH_data["ph"].values - # # pH_lt_6_indices = np.where( (values < 6) | (values > 8.5))[0] - # pH_lt_6_indices = np.where( (values < 6) )[0] - # windows = list() - # windows.append([pH_lt_6_indices[0]]) - # left = pH_lt_6_indices[0] - # for ii in range(1, pH_lt_6_indices.shape[0]): - # i = pH_lt_6_indices[ii] - # if i - left > 5: - # windows[-1].append(left) - # windows.append([i]) - # left = i - # windows[-1].append(pH_lt_6_indices[-1]) - # for window in windows: - # time_window = [pH_data["ph"].index.values[window[0]], pH_data["ph"].index.values[window[1]]] - # for k in data: - # data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])] + # print data["PH_ATLAS"] + min_ph = ph_bounds[0] + max_ph = ph_bounds[1] + if "PH_ATLAS" in data: + print "pH sensor is present. Trimming all data within PH = ["+str(min_ph)+", "+str(max_ph)+ "] time windows\n" + PH_data = data["PH_ATLAS"] + values = PH_data["ph"].values + + ph_outofbounds_indices = np.where( (values < min_ph) | (values > max_ph) )[0] # | out_of_bouds_lat | out_of_bouds_lon )[0] + + if (len(ph_outofbounds_indices) == 0): + print("no sensor values to prune. all values between "+ str(min_ph) +" and " + str(max_ph)) + else: + windows = list() + windows.append([ph_outofbounds_indices[0]]) + left = ph_outofbounds_indices[0] + for ii in range(1, ph_outofbounds_indices.shape[0]): + i = ph_outofbounds_indices[ii] + if i - left > 5: + # there has been a jump in index, a new time window has started + windows[-1].append(left) + windows.append([i]) + left = i + windows[-1].append(ph_outofbounds_indices[-1]) + # print ph_outofbounds_indices + # print windows + for window in windows: + time_window = [PH_data["ph"].index.values[window[0]], PH_data["ph"].index.values[window[1]]] + for k in data: + data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])] + else: + print "No PH sensor present. No trimming will be performed." # Print the available sensors and channels for this logfile. print "Available sensors/channels:" @@ -191,7 +250,8 @@ def generate_overlay(log_file, sensor_id, min_ec): continue for c in data[s].dtypes.keys(): print " {:s}, {:s}".format(s, str(c)) - + + position = pose[['latitude', 'longitude']] # Extract the pose timing and the sensor data of interest. pose_times = pose.index.values.astype(np.float64) @@ -212,11 +272,6 @@ def generate_overlay(log_file, sensor_id, min_ec): sensor_valid = np.all(np.isfinite(sensor), axis=1) sensor = sensor[sensor_valid] - # Create a trail of the vehicle's path on the map. - pl = Polyline(locations=position.as_matrix().tolist()) - pl.fill_opacity = 0.0 - pl.weight = 2 - ## Add a data overlay for the map data_padding = [0.0000001, 0.0000001] # degrees lat/lon data_resolution = [0.00001, 0.00001] # degrees lat/lon @@ -224,6 +279,7 @@ def generate_overlay(log_file, sensor_id, min_ec): data_bounds = [(position.min() - data_padding).tolist(), (position.max() + data_padding).tolist()] + # print data print position.min() print position.max() print data_bounds @@ -239,6 +295,9 @@ def generate_overlay(log_file, sensor_id, min_ec): print data_shape + data_stats["number_of_points"] = data_shape[0] + data_stats["data_bounds"] = data_bounds + print "starting major processing..." if sensor_name in data: # Create a radial-basis interpolator over the sensor dataset @@ -261,6 +320,8 @@ def generate_overlay(log_file, sensor_id, min_ec): data_max = data_zv[np.isfinite(data_zv)].max() data_min = data_zv[np.isfinite(data_zv)].min() print "Data min = {:f} Data max = {:f}".format(data_min, data_max) + data_stats["data_min"] = data_min + data_stats["data_max"] = data_max NORMALIZER = data_max # 800 data_zv = (data_zv - data_min) / (NORMALIZER - data_min) @@ -277,10 +338,11 @@ def generate_overlay(log_file, sensor_id, min_ec): print "removing file: " + old_png_file os.remove(old_png_file) - out_prefix = log_file + "-"+sensor_name+"-"+str(min_ec)+'-' + out_prefix = log_file + "-"+sensor_name+'-' print "creating overlay files" png_filename_rgb = './overlay/'+out_prefix+'overlay.png' + data_stats["overlay_filename"] = out_prefix+'overlay.png' scipy.misc.imsave(png_filename_rgb, data_rgb) # Create image overlay that references generated image. @@ -305,13 +367,45 @@ def generate_overlay(log_file, sensor_id, min_ec): cb1.set_label(sensor_units) scale_bar_filename = './overlay/'+out_prefix+'bar.png' + data_stats["bar_filename"] = out_prefix+'bar.png' pyplot.savefig(scale_bar_filename) pyplot.close('all') + print data_stats + stats_out = stat_in + stats_out[str(sensor_id)] = data_stats + + with open("./stats/"+log_file+'.json', 'w') as outfile: + json.dump(stats_out, outfile) + if __name__ == '__main__': + if (len(sys.argv) < 3 or (["-h", "help", "h", "--help"] in sys.argv)): + print "args: python data_processor.py log_file_name (w/o .txt appended) sensor_id min_ec" + quit(1) log_file = sys.argv[1] sensor_id = int(sys.argv[2]) min_ec = int(sys.argv[3]) - generate_overlay(log_file, sensor_id, min_ec) + max_ec = int(sys.argv[4]) + min_ph = float(sys.argv[5]) + max_ph = float(sys.argv[6]) + log_path = "/home/shawn/data/ERM/log_files/" + if (os.path.isfile(log_path + log_file + '.txt') or os.path.isfile(log_path + log_file+'.txt.incomplete')): + generate_overlay(log_path, log_file, sensor_id, [min_ec, max_ec], [min_ph, max_ph]) + else: + log_folder = log_file + log_files = [] + for file in os.listdir(log_folder): + if (os.path.isfile(log_folder+"/"+file)): + print "adding file: " + file + log_files.append(os.path.splitext(file)[0]) + else: + print(file +" is not a file") + + print log_files + + for x in log_files: + generate_overlay(log_path, x, sensor_id, [min_ec, max_ec], [min_ph, max_ph]) + + # generate_histogram() diff --git a/image_server/image_server.py b/image_server/image_server.py index 03ef098..28c475f 100644 --- a/image_server/image_server.py +++ b/image_server/image_server.py @@ -13,6 +13,7 @@ import glob import flask from flask import send_from_directory, render_template +import json UPLOAD_FOLDER = '/home/shawn/data/ERM/log_files/' ALLOWED_EXTENSIONS = set(['txt']) @@ -29,13 +30,19 @@ def __init__(self, data): self.data = data -@app.route('/path/') -def send(path): - if (os.path.exists(path)): - print "sending file: " + path - return send_from_directory("", path) - else: - print path +" does not exist" +@app.route('/overlay/') +def serve_static(filename): + print "getting static file: ", filename, "from dir: ",os.path.join(".", 'overlay') + return send_from_directory(os.path.join(".", 'overlay'), filename) + +# @app.route('/overlay/') +# def send(path): +# path = "/overlay/"+path +# if (os.path.exists("./"+path)): +# print "sending file: " + path +# return send_from_directory("overlay", path) +# else: +# print path +" does not exist" def sensor_id_to_name(id): if (id == 0): @@ -58,211 +65,38 @@ def sensor_id_to_name(id): return (sensor_name, sensor_channel, sensor_units) - -@app.route('/logs////overlay.jpg') -def generate_overlay(log_file, sensor_id, min_ec): - # Import the data from the specified logfile - - log_ext = ".txt" - - log_path = "/home/shawn/data/ERM/log_files/" - log_filenames = [ - log_path + log_file - ] - - - data = platypus.io.logs.merge_files(log_filenames) - - # data_store = PlatypusDataStore(data) - - if "EC_DECAGON" in data: - print "ES2 sensor is present. Trimming all data within EC = 0 time windows\n" - # find all time windows where EC is exactly 0 - ES2_data = data["EC_DECAGON"] - values = ES2_data["ec"].values - # ec_eq_zero_indices = np.where(values == 0)[0] - ec_eq_zero_indices = np.where( (values < min_ec))[0] - # ec_eq_zero_indices = np.where(values < 50)[0] - windows = list() - windows.append([ec_eq_zero_indices[0]]) - left = ec_eq_zero_indices[0] - for ii in range(1, ec_eq_zero_indices.shape[0]): - i = ec_eq_zero_indices[ii] - if i - left > 5: - # there has been a jump in index, a new time window has started - windows[-1].append(left) - windows.append([i]) - left = i - windows[-1].append(ec_eq_zero_indices[-1]) - # print ec_eq_zero_indices - # print windows - for window in windows: - time_window = [ES2_data["ec"].index.values[window[0]], ES2_data["ec"].index.values[window[1]]] - for k in data: - data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])] - else: - print "No ES2 sensor present. No trimming will be performed." - - # if "PH_ATLAS" in data: - # print "pH sensor is present. Trimming all data within pH < 6 time windows\n" - # # find all time windows where pH is less than 6 - # pH_data = data["PH_ATLAS"] - # values = pH_data["ph"].values - # # pH_lt_6_indices = np.where( (values < 6) | (values > 8.5))[0] - # pH_lt_6_indices = np.where( (values < 6) )[0] - # windows = list() - # windows.append([pH_lt_6_indices[0]]) - # left = pH_lt_6_indices[0] - # for ii in range(1, pH_lt_6_indices.shape[0]): - # i = pH_lt_6_indices[ii] - # if i - left > 5: - # windows[-1].append(left) - # windows.append([i]) - # left = i - # windows[-1].append(pH_lt_6_indices[-1]) - # for window in windows: - # time_window = [pH_data["ph"].index.values[window[0]], pH_data["ph"].index.values[window[1]]] - # for k in data: - # data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])] - - - # Define useful access variables. - pose = data['pose'] - position = pose[['latitude', 'longitude']] - - # Print the available sensors and channels for this logfile. - print "Available sensors/channels:" - for s in data.keys(): - if s == 'pose' or s == 'BATTERY': - continue - for c in data[s].dtypes.keys(): - print " {:s}, {:s}".format(s, str(c)) - - - # Extract the pose timing and the sensor data of interest. - pose_times = pose.index.values.astype(np.float64) - - if sensor_name in data: - sensor = data[sensor_name] - sensor_times = sensor.index.values.astype(np.float64) - - # Linearly interpolate the position of the sensor at every sample. - sensor_pose_interpolator = scipy.interpolate.interp1d(pose_times, position, - axis=0, bounds_error=False) - - # Add the position information back to the sensor data. - sensor = sensor.join(pandas.DataFrame(sensor_pose_interpolator(sensor_times), sensor.index, - columns=('latitude', 'longitude'))) - - # Remove columns that have NaN values (no pose information). - sensor_valid = np.all(np.isfinite(sensor), axis=1) - sensor = sensor[sensor_valid] - - # Create a trail of the vehicle's path on the map. - pl = Polyline(locations=position.as_matrix().tolist()) - pl.fill_opacity = 0.0 - pl.weight = 2 - - ## Add a data overlay for the map - data_padding = [0.00001, 0.00001] # degrees lat/lon - data_resolution = [0.00001, 0.00001] # degrees lat/lon - data_interpolation_radius = 0.00001 # degrees lat/lon - data_bounds = [(position.min() - data_padding).tolist(), - (position.max() + data_padding).tolist()] - - print position.min() - print position.max() - print data_bounds - print data_resolution - - # Create a rectangular grid of overlay points. - data_xv, data_yv = np.meshgrid( - np.arange(data_bounds[1][0], data_bounds[0][0], -data_resolution[0]), - np.arange(data_bounds[0][1], data_bounds[1][1], data_resolution[1]) - ) - data_shape = data_xv.shape - data_xy = np.vstack([data_xv.ravel(), data_yv.ravel()]).T - - if sensor_name in data: - # Create a radial-basis interpolator over the sensor dataset - # Then, query it at each point of the rectangular grid. - #from sklearn.neighbors import RadiusNeighborsClassifier - #data_estimator = RadiusNeighborsClassifier(radius=data_interpolation_radius, outlier_label=np.nan) - from sklearn.neighbors import RadiusNeighborsRegressor - data_estimator = RadiusNeighborsRegressor(radius=data_interpolation_radius) - - data_estimator.fit(sensor[['latitude','longitude']], sensor[sensor_channel].astype(np.float)) - data_zv = data_estimator.predict(data_xy) - data_zv = data_zv.reshape(data_shape).T - - # Normalize data from [0, 1) - data_max = data_zv[np.isfinite(data_zv)].max() - data_min = data_zv[np.isfinite(data_zv)].min() - print "Data min = {:f} Data max = {:f}".format(data_min, data_max) - NORMALIZER = data_max # 800 - data_zv = (data_zv - data_min) / (NORMALIZER - data_min) - - - - if sensor_name in data: - # Update a color map only at the points that have valid values. - data_rgb = np.zeros((data_shape[0], data_shape[1], 4), dtype=np.uint8) - data_rgb = matplotlib.cm.jet(data_zv) * 255 - data_rgb[:,:,3] = 255 * np.isfinite(data_zv) - - # Remove any old image files. - old_png_files = glob.glob('./*.png') - for old_png_file in old_png_files: - print "removing file: " + old_png_file - os.remove(old_png_file) - - png_filename_rgb = './platypus_data_{:s}.png'.format(uuid.uuid4()) - scipy.misc.imsave(png_filename_rgb, data_rgb) - - # Create image overlay that references generated image. - # makes an image w/ a slight opaque tint to it - data_overlay_tint = np.ones((data_shape[0], data_shape[1], 4), dtype=np.uint8) * 50 - data_overlay_tint_filename = './platypus_data_{:s}.png'.format(uuid.uuid4()) - scipy.misc.imsave(data_overlay_tint_filename, data_overlay_tint) - - data_bounds_tint = [(position.min() - [x * 500 for x in data_padding]).tolist(), - (position.max() + [x * 500 for x in data_padding]).tolist()] - io_blank = ImageOverlay(url=data_overlay_tint_filename, bounds=data_bounds_tint) - io = ImageOverlay(url=png_filename_rgb, bounds=data_bounds) - - # Create a map centered on this data log. - center = [pose['latitude'].median(), pose['longitude'].median()] - # print center - zoom = 17 - # m = Map(center=center, zoom=zoom, height='1300px') - # if sensor_name in data: - # m += io_blank - # m += io # Add image overlay - # if sensor_name not in data: - # m += pl # Add vehicle trail, but only if there isn't heatmap data to look at - - # Make a figure and axes with dimensions as desired. - fig = pyplot.figure(figsize=(15, 3)) - ax1 = fig.add_axes([0.05, 0.80, 0.9, 0.15]) - - if sensor_name in data: - # Set the colormap and norm to correspond to the data for which - # the colorbar will be used. - cmap = matplotlib.cm.jet - norm = matplotlib.colors.Normalize(vmin=data_min, vmax=NORMALIZER) - cb1 = matplotlib.colorbar.ColorbarBase(ax1, cmap=cmap, - norm=norm, - orientation='horizontal') - cb1.set_label(sensor_units) - - scale_bar_filename = './platypus_data_{:s}.png'.format(uuid.uuid4()) - pyplot.savefig(scale_bar_filename) - pyplot.close('all') - # pyplot.show() - return render_template('index.html', bar = scale_bar_filename, map_overlay = png_filename_rgb, log_filenames = log_filenames, center=center, zoom=zoom, data_bounds=data_bounds) +@app.route('/logs//') +def render_page(log_file, sensor_id): + print "rendering log file page from stats" + # open stats file for this log + with open("./stats/"+log_file+'.json', 'r') as readfile: + data = json.load(readfile) + print data + data = data[str(sensor_id)] + + print data + data["bar_filename"] = "/overlay/"+data["bar_filename"] + data["overlay_filename"] = "/overlay/"+data["overlay_filename"] + return render_template('render.html', log_file = log_file, data_bounds = data["data_bounds"], bar = data["bar_filename"], map_overlay = data["overlay_filename"], data_min = data["data_min"], data_max = data["data_max"]) + +@app.route('/index') +@app.route('/') +def render_index(): + log_folder = "/home/shawn/data/ERM/log_files/" + log_files = [] + for file in os.listdir(log_folder): + if (os.path.isfile(log_folder+file)): + print "adding file: " + file + log_files.append(os.path.splitext(file)[0]) + else: + print(file +" is not a file") + + print log_files + + return render_template('index.html', log_folder = log_folder, log_files = log_files) def root_dir(): # pragma: no cover return os.path.abspath(os.path.dirname(__file__)) if __name__ == '__main__': - main() + app.run() \ No newline at end of file diff --git a/image_server/index.html b/image_server/index.html deleted file mode 100644 index 97c999f..0000000 --- a/image_server/index.html +++ /dev/null @@ -1,8 +0,0 @@ - - - -

Hello {{ name }}!

- - - - \ No newline at end of file diff --git a/image_server/templates/index.html b/image_server/templates/index.html index 7c1f2e8..9a2ce5e 100644 --- a/image_server/templates/index.html +++ b/image_server/templates/index.html @@ -2,56 +2,22 @@ - Quick Start - Leaflet + Platypus DataVis - index - - - +

{{ log_folder }}!

- -

{{ log_filenames }}!

- -
- - - - + diff --git a/image_server/templates/render.html b/image_server/templates/render.html new file mode 100644 index 0000000..d0d2892 --- /dev/null +++ b/image_server/templates/render.html @@ -0,0 +1,56 @@ + + + + + Quick Start - Leaflet + + + + + + + + + + + + +

HOME

+

{{ log_file }}!

+ +
+ + + + + + + From 75dff2d3e73c08431f3ba63110c2bf8a195b3b0f Mon Sep 17 00:00:00 2001 From: Shawn Hanna Date: Tue, 10 Jul 2018 18:11:01 -0400 Subject: [PATCH 34/43] a few more updates to the server --- image_server/data_processor.py | 116 ++++++++++++++++++----------- image_server/image_server.py | 4 +- image_server/templates/index.html | 2 +- image_server/templates/render.html | 4 +- 4 files changed, 78 insertions(+), 48 deletions(-) diff --git a/image_server/data_processor.py b/image_server/data_processor.py index 7076bd8..939e842 100644 --- a/image_server/data_processor.py +++ b/image_server/data_processor.py @@ -34,14 +34,29 @@ def sensor_id_to_name(id): return (sensor_name, sensor_channel, sensor_units) -def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds): +def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds, turbidity_bounds): + print "processing: " + log_file + min_ec = ec_bounds[0] + max_ec = ec_bounds[1] + min_ph = ph_bounds[0] + max_ph = ph_bounds[1] + min_turbidity = turbidity_bounds[0] + max_turbidity = turbidity_bounds[1] + data_padding = [0.0000001, 0.0000001] # degrees lat/lon + data_resolution = [0.00001, 0.00001] # degrees lat/lon + data_interpolation_radius = 0.00001 # degrees lat/lon + + data_boundaries = [[37.751816, -122.384368], [37.764696, -122.366599]] + # read the old generation stats file stat_in = {} + old_stats = {"settings": {}} try: with open("./stats/"+log_file+'.json', 'r') as infile: print "reading previous stats from: " + "./stats/"+log_file+'.json' stats_in = json.load(infile) + old_stats = stat_in[str(sensor_id)] except: print "failed to load from input stats file" @@ -52,6 +67,13 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds): data_stats["settings"]["sensor_id"] = sensor_id data_stats["settings"]["ec_bounds"] = ec_bounds data_stats["settings"]["ph_bounds"] = ph_bounds + data_stats["settings"]["turbidity_bounds"] = turbidity_bounds + + print(str(data_stats["settings"])) + + if (data_stats["settings"] == old_stats["settings"]): + print "old processing settings == new processing settings. don't re-run" + return # Import the data from the specified logfile @@ -65,18 +87,11 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds): print "Error. log does not exist: " + log_path + log_file + log_ext return False - log_filenames = [ - log_path + log_file + log_ext - ] - - data = platypus.io.logs.merge_files(log_filenames) + data = platypus.io.logs.load(log_path+log_file+log_ext) (sensor_name, sensor_channel, sensor_units) = sensor_id_to_name(sensor_id) # Define useful access variables. - pose = data['pose'] - - data_boundaries = [ [37.751816, -122.384368], [37.764696, -122.366599]] if (data_boundaries != []): print "Trimming all data within long/lat = "+str(data_boundaries) # find all time windows where EC is exactly 0 @@ -86,8 +101,14 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds): values_lon = ES2_data["longitude"].values # ec_eq_zero_indices = np.where(values == 0)[0] - ec_eq_zero_indices = np.where( (values_lat < data_boundaries[0][0]) | (values_lat > data_boundaries[1][0]) | - (values_lon < data_boundaries[0][1]) | (values_lon > data_boundaries[1][1]) )[0] + lat_min = min(data_boundaries[0][0], data_boundaries[1][0]) + lat_max = max(data_boundaries[0][0], data_boundaries[1][0]) + lon_min = min(data_boundaries[0][1], data_boundaries[1][1]) + lon_max = max(data_boundaries[0][1], data_boundaries[1][1]) + print lat_min, lat_max, " | ", lon_min, lon_max + ec_eq_zero_indices = np.where( (values_lat < lat_min) | (values_lat > lat_max) | + (values_lon < lon_min) | (values_lon > lon_max) + )[0] if (len(ec_eq_zero_indices) == 0): print "no poses outside the bounds" @@ -177,10 +198,6 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds): # else: # print "No ES2 sensor present. No trimming will be performed." - - min_ec = ec_bounds[0] - max_ec = ec_bounds[1] - if "EC_DECAGON" in data: print "ES2 sensor is present. Trimming all data within EC = "+str(min_ec)+" time windows\n" # find all time windows where EC is exactly 0 @@ -189,30 +206,29 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds): # ec_eq_zero_indices = np.where(values == 0)[0] ec_eq_zero_indices = np.where( (values < min_ec) | (values > max_ec) )[0] # | out_of_bouds_lat | out_of_bouds_lon )[0] - # ec_eq_zero_indices = np.where(values < 50)[0] - windows = list() - windows.append([ec_eq_zero_indices[0]]) - left = ec_eq_zero_indices[0] - for ii in range(1, ec_eq_zero_indices.shape[0]): - i = ec_eq_zero_indices[ii] - if i - left > 5: - # there has been a jump in index, a new time window has started - windows[-1].append(left) - windows.append([i]) - left = i - windows[-1].append(ec_eq_zero_indices[-1]) - # print ec_eq_zero_indices - # print windows - for window in windows: - time_window = [ES2_data["ec"].index.values[window[0]], ES2_data["ec"].index.values[window[1]]] - for k in data: - data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])] + if (len(ec_eq_zero_indices) == 0): + print "no ec data to trim" + else: + windows = list() + windows.append([ec_eq_zero_indices[0]]) + left = ec_eq_zero_indices[0] + for ii in range(1, ec_eq_zero_indices.shape[0]): + i = ec_eq_zero_indices[ii] + if i - left > 5: + # there has been a jump in index, a new time window has started + windows[-1].append(left) + windows.append([i]) + left = i + windows[-1].append(ec_eq_zero_indices[-1]) + # print ec_eq_zero_indices + # print windows + for window in windows: + time_window = [ES2_data["ec"].index.values[window[0]], ES2_data["ec"].index.values[window[1]]] + for k in data: + data[k] = data[k].loc[np.logical_or(data[k].index < time_window[0], data[k].index > time_window[1])] else: print "No ES2 sensor present. No trimming will be performed." - # print data["PH_ATLAS"] - min_ph = ph_bounds[0] - max_ph = ph_bounds[1] if "PH_ATLAS" in data: print "pH sensor is present. Trimming all data within PH = ["+str(min_ph)+", "+str(max_ph)+ "] time windows\n" PH_data = data["PH_ATLAS"] @@ -251,6 +267,7 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds): for c in data[s].dtypes.keys(): print " {:s}, {:s}".format(s, str(c)) + pose = data['pose'] position = pose[['latitude', 'longitude']] # Extract the pose timing and the sensor data of interest. @@ -273,9 +290,6 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds): sensor = sensor[sensor_valid] ## Add a data overlay for the map - data_padding = [0.0000001, 0.0000001] # degrees lat/lon - data_resolution = [0.00001, 0.00001] # degrees lat/lon - data_interpolation_radius = 0.00001 # degrees lat/lon data_bounds = [(position.min() - data_padding).tolist(), (position.max() + data_padding).tolist()] @@ -322,8 +336,20 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds): print "Data min = {:f} Data max = {:f}".format(data_min, data_max) data_stats["data_min"] = data_min data_stats["data_max"] = data_max - NORMALIZER = data_max # 800 - data_zv = (data_zv - data_min) / (NORMALIZER - data_min) + if (sensor_id == 0): + # pH + color_data_min = 5 + color_data_max = 10 + elif (sensor_id == 1): + # ec + color_data_min = 30000 + color_data_max = 95000 + elif (sensor_id == 3): + # turbidity (DO_ATLAS) + color_data_min = 0 + color_data_max = 1000 + NORMALIZER = color_data_max # 800 + data_zv = (data_zv - color_data_min) / (NORMALIZER - color_data_min) print "create color map" @@ -360,7 +386,7 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds): # Set the colormap and norm to correspond to the data for which # the colorbar will be used. cmap = matplotlib.cm.jet - norm = matplotlib.colors.Normalize(vmin=data_min, vmax=NORMALIZER) + norm = matplotlib.colors.Normalize(vmin=color_data_min, vmax=NORMALIZER) cb1 = matplotlib.colorbar.ColorbarBase(ax1, cmap=cmap, norm=norm, orientation='horizontal') @@ -389,9 +415,11 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds): max_ec = int(sys.argv[4]) min_ph = float(sys.argv[5]) max_ph = float(sys.argv[6]) + min_turbidity = 0 + max_turbidity = 1000 log_path = "/home/shawn/data/ERM/log_files/" if (os.path.isfile(log_path + log_file + '.txt') or os.path.isfile(log_path + log_file+'.txt.incomplete')): - generate_overlay(log_path, log_file, sensor_id, [min_ec, max_ec], [min_ph, max_ph]) + generate_overlay(log_path, log_file, sensor_id, [min_ec, max_ec], [min_ph, max_ph], [min_turbidity, max_turbidity]) else: log_folder = log_file log_files = [] @@ -405,7 +433,7 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds): print log_files for x in log_files: - generate_overlay(log_path, x, sensor_id, [min_ec, max_ec], [min_ph, max_ph]) + generate_overlay(log_path, x, sensor_id, [min_ec, max_ec], [min_ph, max_ph], [min_turbidity, max_turbidity]) # generate_histogram() diff --git a/image_server/image_server.py b/image_server/image_server.py index 28c475f..8d960bb 100644 --- a/image_server/image_server.py +++ b/image_server/image_server.py @@ -77,7 +77,8 @@ def render_page(log_file, sensor_id): print data data["bar_filename"] = "/overlay/"+data["bar_filename"] data["overlay_filename"] = "/overlay/"+data["overlay_filename"] - return render_template('render.html', log_file = log_file, data_bounds = data["data_bounds"], bar = data["bar_filename"], map_overlay = data["overlay_filename"], data_min = data["data_min"], data_max = data["data_max"]) + (sensor_name, sensor_channel, sensor_units) = sensor_id_to_name(sensor_id) + return render_template('render.html', log_file = log_file, data_bounds = data["data_bounds"], bar = data["bar_filename"], map_overlay = data["overlay_filename"], data_min = data["data_min"], data_max = data["data_max"], sensor_name = sensor_name) @app.route('/index') @app.route('/') @@ -91,6 +92,7 @@ def render_index(): else: print(file +" is not a file") + log_files.sort() print log_files return render_template('index.html', log_folder = log_folder, log_files = log_files) diff --git a/image_server/templates/index.html b/image_server/templates/index.html index 9a2ce5e..4b5f6b5 100644 --- a/image_server/templates/index.html +++ b/image_server/templates/index.html @@ -15,7 +15,7 @@

{{ log_folder }}!

diff --git a/image_server/templates/render.html b/image_server/templates/render.html index d0d2892..f0e7473 100644 --- a/image_server/templates/render.html +++ b/image_server/templates/render.html @@ -15,8 +15,8 @@ -

HOME

-

{{ log_file }}!

+

Data Home

+

{{ log_file }} - {{ sensor_name }}

From 7be809842babf4f67828f49c964d95852568349a Mon Sep 17 00:00:00 2001 From: Shawn Hanna Date: Tue, 10 Jul 2018 22:52:36 -0400 Subject: [PATCH 35/43] finished up the initial server pipeline --- image_server/data_processor.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/image_server/data_processor.py b/image_server/data_processor.py index 939e842..549ae0d 100644 --- a/image_server/data_processor.py +++ b/image_server/data_processor.py @@ -49,14 +49,15 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds, turbid data_boundaries = [[37.751816, -122.384368], [37.764696, -122.366599]] # read the old generation stats file - stat_in = {} + stats_in = {} old_stats = {"settings": {}} try: with open("./stats/"+log_file+'.json', 'r') as infile: print "reading previous stats from: " + "./stats/"+log_file+'.json' stats_in = json.load(infile) - old_stats = stat_in[str(sensor_id)] + print stats_in + # old_stats = stats_in[str(sensor_id)] except: print "failed to load from input stats file" @@ -270,6 +271,8 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds, turbid pose = data['pose'] position = pose[['latitude', 'longitude']] + # print data + # Extract the pose timing and the sensor data of interest. pose_times = pose.index.values.astype(np.float64) @@ -344,6 +347,10 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds, turbid # ec color_data_min = 30000 color_data_max = 95000 + elif (sensor_id == 2): + # ec + color_data_min = 5 + color_data_max = 30 elif (sensor_id == 3): # turbidity (DO_ATLAS) color_data_min = 0 @@ -399,11 +406,11 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds, turbid pyplot.close('all') print data_stats - stats_out = stat_in + stats_out = stats_in stats_out[str(sensor_id)] = data_stats with open("./stats/"+log_file+'.json', 'w') as outfile: - json.dump(stats_out, outfile) + json.dump(stats_out, outfile, sort_keys=True, indent=4) if __name__ == '__main__': if (len(sys.argv) < 3 or (["-h", "help", "h", "--help"] in sys.argv)): @@ -419,7 +426,11 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds, turbid max_turbidity = 1000 log_path = "/home/shawn/data/ERM/log_files/" if (os.path.isfile(log_path + log_file + '.txt') or os.path.isfile(log_path + log_file+'.txt.incomplete')): - generate_overlay(log_path, log_file, sensor_id, [min_ec, max_ec], [min_ph, max_ph], [min_turbidity, max_turbidity]) + if sensor_id == -1: + for x in range(0, 3): + generate_overlay(log_path, log_file, x, [min_ec, max_ec], [min_ph, max_ph], [min_turbidity, max_turbidity]) + else: + generate_overlay(log_path, log_file, sensor_id, [min_ec, max_ec], [min_ph, max_ph], [min_turbidity, max_turbidity]) else: log_folder = log_file log_files = [] @@ -433,7 +444,11 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds, turbid print log_files for x in log_files: - generate_overlay(log_path, x, sensor_id, [min_ec, max_ec], [min_ph, max_ph], [min_turbidity, max_turbidity]) + if (sensor_id == -1): + for y in range(0, 3): + generate_overlay(log_path, x, y, [min_ec, max_ec], [min_ph, max_ph], [min_turbidity, max_turbidity]) + else: + generate_overlay(log_path, x, sensor_id, [min_ec, max_ec], [min_ph, max_ph], [min_turbidity, max_turbidity]) # generate_histogram() From 62fa18dfca2e38c1aafb4642c7d54ec76b978d08 Mon Sep 17 00:00:00 2001 From: Shawn Hanna Date: Thu, 12 Jul 2018 00:40:23 -0400 Subject: [PATCH 36/43] fixed range issue with processing script so it now processes all possible sensors --- image_server/.gitignore | 3 +++ image_server/data_processor.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 image_server/.gitignore diff --git a/image_server/.gitignore b/image_server/.gitignore new file mode 100644 index 0000000..3da24a9 --- /dev/null +++ b/image_server/.gitignore @@ -0,0 +1,3 @@ +overlay/* +stats/* + diff --git a/image_server/data_processor.py b/image_server/data_processor.py index 549ae0d..1dec434 100644 --- a/image_server/data_processor.py +++ b/image_server/data_processor.py @@ -427,7 +427,7 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds, turbid log_path = "/home/shawn/data/ERM/log_files/" if (os.path.isfile(log_path + log_file + '.txt') or os.path.isfile(log_path + log_file+'.txt.incomplete')): if sensor_id == -1: - for x in range(0, 3): + for x in range(0, 4): generate_overlay(log_path, log_file, x, [min_ec, max_ec], [min_ph, max_ph], [min_turbidity, max_turbidity]) else: generate_overlay(log_path, log_file, sensor_id, [min_ec, max_ec], [min_ph, max_ph], [min_turbidity, max_turbidity]) @@ -445,7 +445,7 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds, turbid for x in log_files: if (sensor_id == -1): - for y in range(0, 3): + for y in range(0, 4): generate_overlay(log_path, x, y, [min_ec, max_ec], [min_ph, max_ph], [min_turbidity, max_turbidity]) else: generate_overlay(log_path, x, sensor_id, [min_ec, max_ec], [min_ph, max_ph], [min_turbidity, max_turbidity]) From d572b4d667788de3afb2eaac819cf483be1eee9f Mon Sep 17 00:00:00 2001 From: Shawn Hanna Date: Mon, 16 Jul 2018 06:23:39 +0000 Subject: [PATCH 37/43] updates for AWS usage of server --- image_server/data_processor.py | 11 +++++++++-- image_server/image_server.py | 7 ++++--- image_server/overlay/.gitignore | 4 ++++ image_server/stats/.gitignore | 4 ++++ 4 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 image_server/overlay/.gitignore create mode 100644 image_server/stats/.gitignore diff --git a/image_server/data_processor.py b/image_server/data_processor.py index 1dec434..03fb5e2 100644 --- a/image_server/data_processor.py +++ b/image_server/data_processor.py @@ -1,5 +1,7 @@ from sklearn.neighbors import RadiusNeighborsRegressor import matplotlib +matplotlib.use('Agg') + import matplotlib.cm from matplotlib import pyplot import numpy as np @@ -13,6 +15,8 @@ import sys import json +log_path = '/home/ubuntu/mount_nas/' + def sensor_id_to_name(id): if (id == 0): # Select the sensor and the name of the channel for that sensor. @@ -411,6 +415,8 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds, turbid with open("./stats/"+log_file+'.json', 'w') as outfile: json.dump(stats_out, outfile, sort_keys=True, indent=4) + else: + print "sensor name: " + sensor_name +" is not in:\n", data if __name__ == '__main__': if (len(sys.argv) < 3 or (["-h", "help", "h", "--help"] in sys.argv)): @@ -424,11 +430,11 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds, turbid max_ph = float(sys.argv[6]) min_turbidity = 0 max_turbidity = 1000 - log_path = "/home/shawn/data/ERM/log_files/" if (os.path.isfile(log_path + log_file + '.txt') or os.path.isfile(log_path + log_file+'.txt.incomplete')): if sensor_id == -1: for x in range(0, 4): generate_overlay(log_path, log_file, x, [min_ec, max_ec], [min_ph, max_ph], [min_turbidity, max_turbidity]) + print "\n\n\n\n\n\n\n\n\n" else: generate_overlay(log_path, log_file, sensor_id, [min_ec, max_ec], [min_ph, max_ph], [min_turbidity, max_turbidity]) else: @@ -447,8 +453,9 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds, turbid if (sensor_id == -1): for y in range(0, 4): generate_overlay(log_path, x, y, [min_ec, max_ec], [min_ph, max_ph], [min_turbidity, max_turbidity]) + print "\n\n\n\n\n\n\n\n\n" else: generate_overlay(log_path, x, sensor_id, [min_ec, max_ec], [min_ph, max_ph], [min_turbidity, max_turbidity]) - + print "\n\n\n\n\n\n\n\n\n" # generate_histogram() diff --git a/image_server/image_server.py b/image_server/image_server.py index 8d960bb..87bcff0 100644 --- a/image_server/image_server.py +++ b/image_server/image_server.py @@ -1,4 +1,3 @@ -from ipyleaflet import Map, ImageOverlay, Polyline import matplotlib import matplotlib.cm from matplotlib import pyplot @@ -83,7 +82,7 @@ def render_page(log_file, sensor_id): @app.route('/index') @app.route('/') def render_index(): - log_folder = "/home/shawn/data/ERM/log_files/" + log_folder = "./stats/" log_files = [] for file in os.listdir(log_folder): if (os.path.isfile(log_folder+file)): @@ -101,4 +100,6 @@ def root_dir(): # pragma: no cover return os.path.abspath(os.path.dirname(__file__)) if __name__ == '__main__': - app.run() \ No newline at end of file + app.debug = True + app.run(host='0.0.0.0') + #app.run() diff --git a/image_server/overlay/.gitignore b/image_server/overlay/.gitignore new file mode 100644 index 0000000..5e7d273 --- /dev/null +++ b/image_server/overlay/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/image_server/stats/.gitignore b/image_server/stats/.gitignore new file mode 100644 index 0000000..5e7d273 --- /dev/null +++ b/image_server/stats/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore From 98045d993b7e3286b40154307d5ee3f47c9c490c Mon Sep 17 00:00:00 2001 From: Shawn Hanna Date: Mon, 16 Jul 2018 06:38:24 +0000 Subject: [PATCH 38/43] final changes for v0.1 --- image_server/image_server.py | 2 ++ image_server/run_periodic.sh | 6 ++++++ image_server/templates/index.html | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 image_server/run_periodic.sh diff --git a/image_server/image_server.py b/image_server/image_server.py index 87bcff0..e5825f8 100644 --- a/image_server/image_server.py +++ b/image_server/image_server.py @@ -85,6 +85,8 @@ def render_index(): log_folder = "./stats/" log_files = [] for file in os.listdir(log_folder): + if (str(file) == ".gitignore"): + continue if (os.path.isfile(log_folder+file)): print "adding file: " + file log_files.append(os.path.splitext(file)[0]) diff --git a/image_server/run_periodic.sh b/image_server/run_periodic.sh new file mode 100644 index 0000000..c1e18a9 --- /dev/null +++ b/image_server/run_periodic.sh @@ -0,0 +1,6 @@ +#!/bin/bash +while [ 1 ] +do + python data_processor.py ~/mount_nas/ -1 5000 1000000 5 12 + sleep 60 +done diff --git a/image_server/templates/index.html b/image_server/templates/index.html index 4b5f6b5..2657842 100644 --- a/image_server/templates/index.html +++ b/image_server/templates/index.html @@ -11,7 +11,7 @@ -

{{ log_folder }}!

+

ERM Robotic Boat Sensor Data

    {% for file in log_files %} From 83931b08e180757db2fb636d71c35eca3cc141b7 Mon Sep 17 00:00:00 2001 From: Shawn Hanna Date: Wed, 25 Jul 2018 05:56:26 +0000 Subject: [PATCH 39/43] added links to other data types in each render page --- image_server/templates/render.html | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/image_server/templates/render.html b/image_server/templates/render.html index f0e7473..74ae589 100644 --- a/image_server/templates/render.html +++ b/image_server/templates/render.html @@ -14,12 +14,13 @@ -

    Data Home

    {{ log_file }} - {{ sensor_name }}

    -
    - +This run's pH data | This run's EC data | This run's Temperature data | This run's Turbidity data

    + +
    + + + + + + + + -

    Data Home

    -

    {{ log_file }} - {{ sensor_name }}

    +

    Data Home

    +

    {{ log_file }} - {{ sensor_name }}

    + +
    + This run's raw {{ sensor_name }} data +
    + + This run's pH data | This run's EC data | This run's Temperature data | This run's Turbidity data

    + +
    + + -
    -This run's raw {{ sensor_name }} data -
    + + var bounds_poly = L.rectangle([bounds.pad(0.05).getSouthWest(),bounds.pad(0.05).getNorthEast()], + {color: 'red', interactive: false, weight: 1, fill: false} + ).addTo(map); + - +
    + REPROCESS DATA + From 48e5d4fb7299af72930ce79b9d04d1f885537425 Mon Sep 17 00:00:00 2001 From: Shawn Hanna Date: Wed, 10 Oct 2018 22:27:37 +0000 Subject: [PATCH 43/43] updates to ERM scripts --- image_server/data_processor.py | 7 +++++-- image_server/run_periodic.sh | 26 +++++++++++++++++--------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/image_server/data_processor.py b/image_server/data_processor.py index 5608423..a5aa59d 100644 --- a/image_server/data_processor.py +++ b/image_server/data_processor.py @@ -17,7 +17,7 @@ import sys import json -log_path = '/home/ubuntu/mount_nas/' +log_path = '/home/ubuntu/process_ERM/' def sensor_id_to_name(id): if (id == 0): @@ -53,6 +53,7 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds, turbid data_interpolation_radius = 0.00001 # degrees lat/lon data_boundaries = [[37.756664, -122.381500], [37.760387, -122.377216]] +# data_boundaries = [] # read the old generation stats file stats_in = {} @@ -284,6 +285,8 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds, turbid pose_times = pose.index.values.astype(np.float64) if sensor_name in data: + if (sensor_name == "DO_ATLAS"): + data[sensor_name]["do"] = data[sensor_name]["do"]/20.0 sensor = data[sensor_name] sensor_times = sensor.index.values.astype(np.float64) @@ -368,7 +371,7 @@ def generate_overlay(log_path, log_file, sensor_id, ec_bounds, ph_bounds, turbid # turbidity (DO_ATLAS) num_bins = 40 color_data_min = 0 - color_data_max = 1000 + color_data_max = 50 data_stats["data_stddev"] = data[sensor_name][sensor_channel].std() data_stats["data_min"] = data[sensor_name][sensor_channel].min() diff --git a/image_server/run_periodic.sh b/image_server/run_periodic.sh index 3a4003f..c2b59fa 100644 --- a/image_server/run_periodic.sh +++ b/image_server/run_periodic.sh @@ -1,19 +1,27 @@ #!/bin/bash while [ 1 ] do - # python data_processor.py ~/mount_nas/ -1 5000 1000000 5 12 + cp -n -r ~/mount_nas/ERM/phone\ \#2/ ~/backup_ERM + cp -n -r ~/mount_nas/ERM/phone\ \#1/ ~/backup_ERM + cp -n ~/mount_nas/ERM/*txt* ~/backup_ERM + + rm -r ~/process_ERM/ + mkdir -p ~/process_ERM/ + cp -n ~/mount_nas/ERM/phone\ \#2/* ~/process_ERM/ + cp -n ~/mount_nas/ERM/phone\ \#1/* ~/process_ERM/ + cp -n ~/mount_nas/ERM/*txt* ~/process_ERM/ - for filename in ~/mount_nas/*.txt; do + for filename in ~/process_ERM/*; do + if [[ $(find "$filename" -mtime +10 -print) ]]; then + echo "skipping old file: $filename" + # sleep 0.5 + # continue + fi for ((i=0; i<=3; i++)); do python data_processor.py "$filename" "$i" 5000 1000000 6 10.5 - done - done + done - for filename in "~/mount_nas/phone 2/*.txt"; do - for ((i=0; i<=3; i++)); do - python data_processor.py "$filename" "$i" 5000 1000000 6 10.5 - done done - + echo "Done for now" sleep 3600 done