Profile code

README.txt

@@ -0,0 +1,34 @@
+#Use these commands inside stats: 1) sort tottime 2) stats 20
+
+(base) ad130774-mlt:datalogger-to-ml snag$ python -m pstats dpmData.profile
+Welcome to the profile statistics browser.
+dpmData.profile% sort tottime
+dpmData.profile% stats 20
+Tue Jun 23 10:12:31 2020    dpmData.profile
+
+         17180935 function calls (17105828 primitive calls) in 52.271 seconds
+
+   Ordered by: internal time
+   List reduced from 4952 to 20 due to restriction <20>
+
+   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
+     2660   14.215    0.005   14.215    0.005 {method '_g_flush' of 'tables.hdf5extension.Leaf' objects}
+     2670    6.175    0.002    6.175    0.002 {tables.indexesextension.keysort}
+  1338042    2.879    0.000    2.879    0.000 dpm_protocol.py:1034(consumeRawInt)
+   101284    2.767    0.000    2.767    0.000 {method '_g_get_objinfo' of 'tables.hdf5extension.Group' objects}
+     2670    2.497    0.001    2.563    0.001 {method '_fill_col' of 'tables.tableextension.Row' objects}
+     7980    2.056    0.000    2.056    0.000 {method '_g_write_slice' of 'tables.hdf5extension.Array' objects}
+    29290    1.937    0.000    2.283    0.000 {method '_g_setattr' of 'tables.hdf5extension.AttributeSet' objects}
+     2670    1.635    0.001    1.848    0.001 /Users/snag/opt/anaconda3/lib/python3.7/site-packages/tables/index.py:636(final_idx32)
+  1295420    1.100    0.000    1.341    0.000 dpm_protocol.py:1077(unmarshal_double)
+  1303405    0.634    0.000    3.458    0.000 dpm_protocol.py:1070(unmarshal_int64)
+     5322    0.465    0.000    5.245    0.001 dpm_protocol.py:1092(<listcomp>)
+    40276    0.464    0.000    0.464    0.000 {method 'reduce' of 'numpy.ufunc' objects}
+1132039/1126508    0.434    0.000    3.929    0.000 {built-in method builtins.getattr}
+     5341    0.378    0.000    0.378    0.000 {pandas._libs.lib.maybe_convert_objects}
+  1206555    0.340    0.000    0.539    0.000 {built-in method builtins.isinstance}
+        3    0.339    0.113    0.339    0.113 {method '_read_records' of 'tables.tableextension.Table' objects}
+95927/95912    0.276    0.000    3.888    0.000 /Users/snag/opt/anaconda3/lib/python3.7/site-packages/tables/group.py:697(_f_get_child)
+     2740    0.250    0.000    0.250    0.000 {method 'acquire' of '_thread.lock' objects}
+   130844    0.245    0.000    0.518    0.000 /Users/snag/opt/anaconda3/lib/python3.7/site-packages/tables/file.py:383(register_node)
+  1295540    0.241    0.000    0.241    0.000 {built-in method _struct.unpack}

dpmData.py

@@ -1,47 +1,83 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """
-Created on Mon Apr 20 11:43:17 2020
+Created on Tue Jun 16 13:20:08 2020
 
 @author: snag
 """
+import argparse
+import datetime
+import sys
 import pandas as pd
 import DPM
 
-DEVICE_LIMIT = 0
-DEVICE_LIST = []
+def main():
+    parser = argparse.ArgumentParser()
+    #Add positional/optional parameters
+    parser.add_argument('-d', '--device_limit', help='Device Limit', type=int, default=0)
+    parser.add_argument('-f', '--device_file', help='device file name')
+    parser.add_argument('-o', '--output_file', help='output file name', default='data.h5')
+    parser.add_argument('-n', '--node', help='node name', default=None, type=str)
+        #Error when not using the default value
+    #group 1
+    parser.add_argument("-s", "--start_date", type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'), help='Enter the start time/date. Do not use the duration tag', required=False)
+    parser.add_argument("-e", "--end_date", type=lambda s: datetime.datetime.strptime(s, '%Y-%m-%d'), help='Enter the start time/date. Do not use the duration tag', required=False)
+    #group 2
+    parser.add_argument("-du", "--duration", help="Enter LOGGERDURATION in sec", required=False)
+    #Parse the args
+    args = parser.parse_args()
+    sys.stdout.write(str(hdf_code(args)))
 
-with open('DeviceList.csv') as f:
-    DEVICE_LIST = [line.rstrip() for index, line in enumerate(f)
-                   if index < DEVICE_LIMIT or DEVICE_LIMIT < 1]
+def hdf_code(args):
+    START_DATE = args.start_date
+    END_DATE = args.end_date
+    DURATION = args.duration
+    DEVICE_LIMIT = args.device_limit
+    DEVICE_FILE = args.device_file
+    OUTPUT_FILE = args.output_file
+    #NODE = args.node #unused
 
-dpm = DPM.Blocking()
-for index, device in enumerate(DEVICE_LIST):
-    dpm.add_entry(index, device)
+    if DURATION and (START_DATE or END_DATE):
+        print("-d and -s|-e are mutually exclusive! Exiting ...")
+        sys.exit(2)
+    elif(START_DATE and END_DATE):
+        DURATION = END_DATE - START_DATE
+        #print(int((END_DATE - START_DATE).total_seconds()))
+        DURATION = str(DURATION)
+        DURATION = 'LOGGERDURATION:' + DURATION
+    elif(DURATION):
+        #Transform duration from int to str, and then add "LOGGERDURATION:" before it.
+        DURATION = str(DURATION)
+        DURATION = 'LOGGERDURATION:' + DURATION
 
-data_done = [None] * len(DEVICE_LIST)
+    DEVICE_LIST = []
+    with open(DEVICE_FILE) as f:
+        DEVICE_LIST = [line.rstrip() for index, line in enumerate(f)
+            if index < DEVICE_LIMIT or DEVICE_LIMIT < 1]
+    dpm = DPM.Blocking()
+    for index, device in enumerate(DEVICE_LIST):
+        dpm.add_entry(index, device)
+    data_done = [None] * len(DEVICE_LIST)
+    hdf = pd.HDFStore(OUTPUT_FILE)
+    for event_response in dpm.process(DURATION):
+        if hasattr(event_response, 'data'):
+            d = {'Timestamps' : event_response.micros, 'Data' : event_response.data}
+            df = pd.DataFrame(data=d)
+            hdf.append(DEVICE_LIST[event_response.tag], df)
+            if len(event_response.data) == 0:
+                data_done[event_response.tag] = True
+        #Status instead of actual data.
+        else:
+            #Want to make it status, but can't because of the bug
+            data_done[event_response.tag] = False
+            print(DEVICE_LIST[event_response.tag], event_response.status)
+        if data_done.count(None) == 0:
+            print(data_done)
+            break
+    #READ THE HDF5 FILE
+    for k in list(hdf.keys()):
+        df = hdf[k]
+        print(k, ':\n', df)
 
-hdf = pd.HDFStore('data.h5')
-for event_response in dpm.process('LOGGERDURATION:360000'):
-    if hasattr(event_response, 'data'):
-        d = {'Timestamps' : event_response.micros, 'Data' : event_response.data}
-        df = pd.DataFrame(data=d)
-        hdf.append(DEVICE_LIST[event_response.tag], df)
-
-        if len(event_response.data) == 0:
-            data_done[event_response.tag] = True
-
-    #Status instead of actual data.
-    else:
-        #Want to make it status, but can't because of the bug
-        data_done[event_response.tag] = False
-        print(DEVICE_LIST[event_response.tag], event_response.status)
-
-    if data_done.count(None) == 0:
-        print(data_done)
-        break
-
-#READ THE HDF5 FILE
-for k in list(hdf.keys()):
-    df = hdf[k]
-    print(k, ':\n', df)
+if __name__ == '__main__':
+    main()