Merge pull request #63 from TACC/57-create-library-for-flags-for-all-…

…the-different-python-scripts

57 create library for flags for all the different python scripts

setup.py

@@ -49,7 +49,7 @@
     include_package_data = True,
     scripts = scripts,
     install_requires = ['argparse','numpy', 'psycopg2-binary', 'pandas', 'pgcopy',
-                        'bokeh', 'django', 'python-hostlist', 'PyMySQL', 'mod_wsgi',
+                        'bokeh', 'django==3.2.25', 'python-hostlist', 'PyMySQL', 'mod_wsgi',
                         'mysql-connector-python', 'python-memcached', 'pika', 'mysqlclient'],
     platforms = 'any',
     classifiers = [

tacc_stats.ini

@@ -1,22 +1,27 @@
-## Basic configuration options - modify these
-# machine	= unique name of machine/queue
-# server	= database and rmq server hostname
-# data_dir	= where data is stored
 [DEFAULT]
-machine		= stampede2
+machine		= machine-name
 data_dir 	= /tacc_stats_site/%(machine)s
-server 		= tacc-stats02.tacc.utexas.edu
+server 		= servername.tacc.utexas.edu
 
-## RabbitMQ Configuration
-# RMQ_SERVER  	= RMQ server
-# RMQ_QUEUE  	= RMQ server
 [RMQ]
 rmq_server 	= %(server)s
 rmq_queue 	= %(machine)s
 
-## Configuration for Web Portal Support
 [PORTAL]
 acct_path       = %(data_dir)s/accounting
 archive_dir	= %(data_dir)s/archive
 host_name_ext   = %(machine)s.tacc.utexas.edu
-dbname          = %(machine)s_db
+dbname          = test_db
+daily_archive_dir = %(data_dir)s/daily_archive
+engine_name     = django.db.backends.postgresql_psycopg2
+username        = user
+password        = password
+host            = localhost
+port            = 5432
+
+[XALT]
+xalt_engine     = none
+xalt_name       = none
+xalt_user       = none
+xalt_password   = none
+xalt_host       = none

tacc_stats/analysis/gen/jid_table.py

@@ -1,13 +1,16 @@
-import time
+import os,sys,time
+# Append your local repository path here:
+# sys.path.append("/home/sg99/tacc_stats")
 import psycopg2
-import tacc_stats.cfg as cfg
+import tacc_stats.conf_parser as cfg
 from tacc_stats.analysis.gen.utils import read_sql
 
 class jid_table:
 
     def __init__(self, jid):
-        CONNECTION = "dbname=ls6_db1 host=localhost user=postgres port=5432"
 
+        CONNECTION = cfg.get_db_connection_string()
+        print(CONNECTION)
         print("Initializing table for job {0}".format(jid))
 
         self.jid = jid
@@ -18,7 +21,7 @@ def __init__(self, jid):
         # Get job accounting data
         acct_data = read_sql("""select * from job_data where jid = '{0}'""".format(jid), self.conj)
         # job_data accounting host names must be converted to fqdn
-        self.acct_host_list = [h + '.' + cfg.host_name_ext for h in acct_data["host_list"].values[0]]
+        self.acct_host_list = [h + '.' + cfg.get_host_name_ext() for h in acct_data["host_list"].values[0]]
 
         self.start_time = acct_data["start_time"].dt.tz_convert('US/Central').dt.tz_localize(None).values[0]
         self.end_time = acct_data["end_time"].dt.tz_convert('US/Central').dt.tz_localize(None).values[0]

tacc_stats/analysis/metrics/update_metrics.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 import os,sys, pwd
+# Append your local repository path here:
+# sys.path.append("/home/sg99/tacc_stats")
 from datetime import timedelta, datetime
 import psycopg2
 
@@ -10,11 +12,11 @@
 from tacc_stats.site.machine.models import job_data, metrics_data
 from tacc_stats.analysis.metrics import metrics
 
-import tacc_stats.cfg as cfg
+import tacc_stats.conf_parser as cfg
 from tacc_stats.progress import progress
 
+CONNECTION = cfg.get_db_connection_string()
 
-CONNECTION = "dbname={0} user=postgres port=5432".format(cfg.dbname)
 query_create_metric_table = """CREATE TABLE IF NOT EXISTS metrics_data (
                                            id     SERIAL PRIMARY KEY,
                                            jid    VARCHAR(32),
@@ -79,4 +81,3 @@ def update_metrics(date, rerun = False):
     while date <= enddate:
         update_metrics(date, rerun = False)
         date += timedelta(days=1)
-

tacc_stats/conf_parser.py

@@ -0,0 +1,97 @@
+import configparser
+import os
+import sys
+import time
+
+# Append your local repository path here:
+# sys.path.append("/home/username/tacc_stats")
+
+cfg = configparser.ConfigParser()
+
+# Append your local repository path here:
+cfg.read('/home/username/tacc_stats/tacc_stats.ini')
+
+def get_db_connection_string():
+    temp_string = "dbname={0} user="+cfg.get('PORTAL', 'username')+" password="+cfg.get('PORTAL', 'password')+" port="+cfg.get('PORTAL', 'port')
+    connection_string = temp_string.format(cfg.get('PORTAL', 'dbname'))
+    return connection_string
+
+def get_db_name():
+    db_name = cfg.get('PORTAL', 'dbname')
+    return db_name
+
+def get_archive_dir_path():
+    archive_dir_path = cfg.get('PORTAL', 'archive_dir')
+    return archive_dir_path
+
+def get_host_name_ext():
+    host_name_ext = cfg.get('PORTAL', 'host_name_ext')
+    return host_name_ext
+
+def get_accounting_path():
+    accounting_path = cfg.get('PORTAL', 'acct_path')
+    return accounting_path
+
+def get_daily_archive_dir_path():
+    daily_archive_dir_path = cfg.get('PORTAL', 'daily_archive_dir')
+    return daily_archive_dir_path
+
+def get_rmq_server():
+    rmq_server = cfg.get('RMQ', 'rmq_server')
+    return rmq_server
+
+def get_rmq_queue():
+    rmq_queue = cfg.get('RMQ', 'rmq_queue')
+    return rmq_queue
+
+def get_machine_name():
+    machine_name = cfg.get('DEFAULT', 'machine')
+    return machine_name
+
+def get_server_name():
+    server_name = cfg.get('DEFAULT', 'server')
+    return server_name
+
+def get_data_dir_path():
+    data_dir_path = cfg.get('DEFAULT', 'data_dir')
+    return data_dir_path
+
+def get_engine_name():
+    engine_name = cfg.get('PORTAL', 'engine_name')
+    return engine_name
+
+def get_username():
+    username = cfg.get('PORTAL', 'username')
+    return username
+
+def get_password():
+    password = cfg.get('PORTAL', 'password')
+    return password
+
+def get_host():
+    host = cfg.get('PORTAL', 'host')
+    return host
+
+def get_port():
+    port = cfg.get('PORTAL', 'port')
+    return port
+
+def get_xalt_engine():
+    xalt_engine = cfg.get('XALT', 'xalt_engine')
+    return xalt_engine
+
+def get_xalt_name():
+    xalt_name = cfg.get('XALT', 'xalt_name')
+    return xalt_name
+
+def get_xalt_user():
+    xalt_user = cfg.get('XALT', 'xalt_user')
+    return xalt_user
+
+def get_xalt_password():
+    xalt_password = cfg.get('XALT', 'xalt_password')
+    return xalt_password
+
+def get_xalt_host():
+    xalt_host = cfg.get('XALT', 'xalt_host')
+    return xalt_host

tacc_stats/dbload/sacct_gen.py

@@ -1,9 +1,11 @@
 import os, sys
+# Append your local repository path here:
+# sys.path.append("/home/sg99/tacc_stats")
 from datetime import timedelta, date, datetime
 from dateutil.parser import parse
-from tacc_stats import cfg
+import tacc_stats.conf_parser as cfg
 
-acct_path = cfg.acct_path
+acct_path = cfg.get_accounting_path()
 
 def daterange(start_date, end_date):
     for n in range(int ((end_date - start_date).days)):
@@ -24,4 +26,3 @@ def daterange(start_date, end_date):
     sacct_command = "/bin/sacct -a -s CA,CD,F,NF,TO -P -X -S " + single_date.strftime("%Y-%m-%d") + " -E " + (single_date + timedelta(1)).strftime("%Y-%m-%d") +" -o JobID,User,Account,Start,End,Submit,Partition,TimeLimit,JobName,State,NNodes,ReqCPUS,NodeList > " + file_name  
     print(sacct_command)
     os.system(sacct_command)
-

tacc_stats/dbload/sync_acct.py

@@ -1,5 +1,9 @@
 #!/usr/bin/env python3
 import os,sys,time
+
+# Append your local repository path here:
+# sys.path.append("/home/sg99/tacc_stats")
+
 from datetime import timedelta, datetime
 
 import psycopg2
@@ -11,9 +15,10 @@
 import hostlist
 
 from tacc_stats.analysis.gen.utils import read_sql
-from tacc_stats import cfg
 
-CONNECTION = "dbname={0} user=postgres port=5432".format(cfg.dbname)
+import tacc_stats.conf_parser as cfg
+
+CONNECTION = cfg.get_db_connection_string()
 
 query_create_jobdata_table = """CREATE TABLE IF NOT EXISTS job_data (
 jid         VARCHAR(32) NOT NULL,
@@ -113,7 +118,7 @@ def sync_acct(acct_file, date_str):
 
         # Parse and convert raw stats files to pandas dataframe
         start = time.time()
-        directory = cfg.acct_path
+        directory = cfg.get_accounting_path()
 
         while startdate <= enddate:            
             for entry in os.scandir(directory):

tacc_stats/dbload/sync_timedb.py

@@ -2,6 +2,8 @@
 import psycopg2
 from pgcopy import CopyManager
 import os, sys, stat
+# Append your local repository path here:
+# sys.path.append("/home/sg99/tacc_stats")
 import multiprocessing
 import itertools
 from multiprocessing import Pool, get_context, Lock, set_start_method
@@ -17,7 +19,8 @@
 from pandas import DataFrame, to_datetime, Timedelta, Timestamp, concat
 
 from tacc_stats.analysis.gen.utils import read_sql
-from tacc_stats import cfg
+
+import tacc_stats.conf_parser as cfg
 
 # archive toggle
 should_archive = True
@@ -28,11 +31,10 @@
 # Thread count for database loading and archival
 thread_count = 8
 
-tgz_archive_dir = "/tacc_stats_site/ls6/tgz_archives/"
-
+tgz_archive_dir = cfg.get_daily_archive_dir_path()
 
 
-CONNECTION = "dbname={0} user=postgres port=5432".format(cfg.dbname)
+CONNECTION = cfg.get_db_connection_string()
 
 amd64_pmc_eventmap = { 0x43ff03 : "FLOPS,W=48", 0x4300c2 : "BRANCH_INST_RETIRED,W=48", 0x4300c3: "BRANCH_INST_RETIRED_MISS,W=48", 
                        0x4308af : "DISPATCH_STALL_CYCLES1,W=48", 0x43ffae :"DISPATCH_STALL_CYCLES0,W=48" }
@@ -373,6 +375,7 @@ def database_startup():
                                                host  VARCHAR(64),
                                                jid   VARCHAR(32),
                                                type  VARCHAR(32),
+					       dev   VARCHAR(64),
                                                event VARCHAR(64),
                                                unit  VARCHAR(16),                                            
                                                value real,
@@ -415,9 +418,9 @@ def database_startup():
         #cur.execute(query_create_hostdata_hypertable)
         #cur.execute(query_create_compression)
 
-    #    cur.execute(query_create_process_table) 
-    #    cur.execute(query_create_process_index)
-        cur.execute("SELECT pg_size_pretty(pg_database_size('{0}'));".format(cfg.dbname))
+        #cur.execute(query_create_process_table) 
+        #cur.execute(query_create_process_index)
+        cur.execute("SELECT pg_size_pretty(pg_database_size('{0}'));".format(cfg.get_db_name()))
         for x in cur.fetchall():
             print("Database Size:", x[0])
         if debug:
@@ -464,7 +467,7 @@ def database_startup():
 
         # Parse and convert raw stats files to pandas dataframe
         start = time.time()
-        directory = cfg.archive_dir
+        directory = cfg.get_archive_dir_path()
 
         stats_files = []
         ar_file_mapping = {}

tacc_stats/listend.py

@@ -2,9 +2,14 @@
 import pika
 import os, sys
 import time
-import tacc_stats.cfg as cfg
+
 from fcntl import flock, LOCK_EX, LOCK_NB
 
+# Append your local repository path here:
+# sys.path.append("/home/sg99/tacc_stats")
+
+import tacc_stats.conf_parser as cfg
+
 def on_message(channel, method_frame, header_frame, body):
 
     try:
@@ -20,7 +25,7 @@ def on_message(channel, method_frame, header_frame, body):
         host = message.split()[2]
 
     #if host == "localhost.localdomain": return
-    host_dir = os.path.join(cfg.archive_dir, host)
+    host_dir = os.path.join(cfg.get_archive_dir_path(), host)
     if not os.path.exists(host_dir):
         os.makedirs(host_dir)
 
@@ -47,10 +52,10 @@ def on_message(channel, method_frame, header_frame, body):
         print("listend is already running")
         sys.exit()
 
-    parameters = pika.ConnectionParameters(cfg.rmq_server)
+    parameters = pika.ConnectionParameters(cfg.get_rmq_server())
     connection = pika.BlockingConnection(parameters)
     channel = connection.channel()
-    channel.basic_consume(cfg.rmq_queue, on_message)
+    channel.basic_consume(cfg.get_rmq_queue(), on_message)
     try:
         channel.start_consuming()
     except KeyboardInterrupt:

tacc_stats/site/machine/update_db.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 import os,sys, pwd
+# Append your local repository path here:
+# sys.path.append("/home/sg99/tacc_stats")
 from datetime import timedelta, datetime
 from dateutil.parser import parse
 from fcntl import flock, LOCK_EX, LOCK_NB
@@ -9,7 +11,7 @@
 from tacc_stats.site.machine.models import Job, Host, Libraries
 from tacc_stats.site.xalt.models import run, join_run_object, lib
 from tacc_stats.analysis.metrics import metrics
-import tacc_stats.cfg as cfg
+import tacc_stats.conf_parser as cfg
 from tacc_stats.progress import progress
 from tacc_stats.daterange import daterange
 import pytz, calendar
@@ -23,7 +25,7 @@ def update_acct(date, rerun = False):
     tz = pytz.timezone('US/Central')
     ctr = 0
 
-    with open(os.path.join(cfg.acct_path, date.strftime("%Y-%m-%d") + '.txt'), encoding = "latin1") as fd:
+    with open(os.path.join(cfg.get_accounting_path(), date.strftime("%Y-%m-%d") + '.txt'), encoding = "latin1") as fd:
         nrecords = sum(1 for record in csv.DictReader(fd))
         fd.seek(0)
 

tacc_stats/site/machine/update_xalt.py

@@ -1,11 +1,13 @@
 #!/usr/bin/env python
 import os,sys
+# Append your local repository path here:
+# sys.path.append("/home/sg99/tacc_stats")
 from datetime import timedelta,datetime
 os.environ['DJANGO_SETTINGS_MODULE']='tacc_stats.site.tacc_stats_site.settings'
 import django
 django.setup()
 from tacc_stats.site.machine import views
-import tacc_stats.cfg as cfg
+import tacc_stats.conf_parser as cfg
 from tacc_stats.site.machine.models import Job, Libraries
 from tacc_stats.site.xalt.models import run, join_run_object, lib
 
@@ -50,4 +52,3 @@ def daterange(start_date, end_date):
             library = Libraries(object_path = object_path, module_name = module_name)
             library.save()
             library.jobs.add(obj)
-