build

#!/usr/bin/env python

import platform, sys, os, time, threading, subprocess, copy, re


config_file = 'config.default'


system = None
dependencies = False
verbose = False
targets = []

global todo, headers, object_deps, file_flags, thread_cflags, thread_ldflags, lock, print_lock, stop
global include_paths
todo, headers, object_deps, file_flags = {}, {}, {}, {}
lock = threading.Lock()
print_lock = threading.Lock()
stop = False

def disp (msg):
  print_lock.acquire()
  print msg
  sys.stdout.flush()
  print_lock.release()
  




# common definitions:

bin_dir = 'bin'
cmd_dir = 'cmd'
lib_dir = 'lib'
src_dir = 'src'
doc_dir = 'doc'
dev_dir = 'dev'

icon = 'icons/icon.o'
icon_dep = 'icons/icon.rc'

cpp_suffix = '.cpp'
h_suffix = '.h'
libname = 'fouts'
config_suffix = ''




include_paths = [ src_dir, cmd_dir ]

# check if we are compiling a separate project:
mrtrix_dir = '.'
build_script = sys.argv[0]
separate_project = False

def get_real_name (path):
  if os.path.islink (path): return os.readlink (path)
  else: return (path)

while os.path.abspath (os.path.dirname (get_real_name (build_script))) != os.path.abspath (mrtrix_dir):
  if verbose and not separate_project:
    print 'compiling separate project against:'
  separate_project = True
  build_script = os.path.normpath (os.path.join (mrtrix_dir, get_real_name (build_script)))
  mrtrix_dir = os.path.dirname (build_script)
  include_paths += [ os.path.join (mrtrix_dir, src_dir) ]
  if verbose: 
    print '    ' + mrtrix_dir

if separate_project:
  lib_dir = os.path.join (mrtrix_dir, lib_dir)
  config_file = os.path.join (mrtrix_dir, config_file)

  if verbose:
    print ''






# parse command-line:
for arg in sys.argv[1:]:
  if   '-verbose'.startswith(arg): verbose = True
  elif '-dependencies'.startswith(arg): dependencies = True
  elif arg[0] == '-':
    print 'unknown command-line option "' + arg + '"'
    sys.stdout.flush()
    sys.exit (1)
  elif arg == 'clean':
    targets = [ 'clean' ]
    break
  elif os.path.isfile (os.path.join (mrtrix_dir, "config." + arg)):
    config_file = os.path.join (mrtrix_dir, "config." + arg)
    config_suffix = '__' + arg
  else: targets.append(arg)





# load configuration file:
try:
  if verbose:
    print 'reading configuration from "' + config_file + '"...'
  exec (open (config_file))
except IOError:
  print '''no configuration file found!
please run "./configure" prior to invoking this script
'''
  sys.exit (1)






# get version info:

if ld_enabled:
  with open (os.path.join (src_dir, 'bts', 'version.h'), 'r') as f:
      for line in f:
        match = re.match('    const size_t FTS_(\w+)_VERSION = (.*);', line)
        if match:
            if match.group(1) == 'MAJOR':
                major = match.group(2)
            elif match.group(1) == 'MINOR':
                minor = match.group(2)
            elif match.group(1) == 'MICRO':
                micro = match.group(2)

  libname += '-' + major + '_' + minor + '_' + micro + config_suffix

  ld_flags = [ '-l' + libname ] + ld_flags
  libname = lib_prefix + libname + lib_suffix

obj_suffix = config_suffix + obj_suffix
exe_suffix = config_suffix + exe_suffix

# Qt4 settings:
#qt4_cflags += [ '-I' + entry for entry in qt4_include_path ]



# other settings:
include_paths += [ lib_dir, cmd_dir ]
cpp_flags += [ '-Wall' ] + [ '-I' + entry for entry in include_paths ]
ld_flags += [ '-L' + lib_dir ]








###########################################################################
#                           TODO list Entry
###########################################################################

class Entry:
  def __init__ (self, name):
    global todo
    if name in todo.keys(): return
    todo[name] = self

    self.name = name
    self.cmd = []
    self.deps = set()
    self.action = 'NA'
    self.timestamp = mtime (self.name)
    self.dep_timestamp = self.timestamp
    self.currently_being_processed = False

    if is_executable (self.name): self.set_executable()
    elif is_icon (self.name): self.set_icon()
    elif is_object (self.name): self.set_object()
    elif is_library (self.name): self.set_library()
    elif is_moc (self.name): self.set_moc()
    elif not os.path.exists (self.name): raise Exception, 'unknown target "' + self.name + '"'

    [ Entry(item) for item in self.deps ]
    dep_timestamp = [ todo[item].dep_timestamp for item in todo.keys() if item in self.deps and not is_library(item) ]
    if len(dep_timestamp): self.dep_timestamp = max(dep_timestamp)
    

  def execute (self):
    if len(self.cmd) > 0: return execute ('[' + self.action + '] ' + self.name, self.cmd)
    else: return None


  def set_executable (self):
    self.action = 'LB'
    if len(exe_suffix) > 0: cc_file = self.name[:-len(exe_suffix)]
    else: cc_file = self.name
    cc_file = os.path.join (cmd_dir, os.sep.join (cc_file.split(os.sep)[1:])) + cpp_suffix
    self.deps = list_cmd_deps(cc_file)

    try: windres
    except NameError: pass
    else: self.deps.add (icon)

    skip = False
    flags = copy.copy (gsl_ldflags)
    if 'T' in file_flags[cc_file]: flags += thread_ldflags
    if 'Q' in file_flags[cc_file]: flags += qt_ldflags

    if not skip: 
      if not os.path.isdir (bin_dir): 
        os.mkdir (bin_dir)

      if not ld_enabled: 
        self.deps = self.deps.union (list_lib_deps())

      self.cmd = fillin (ld, { 
        'LDFLAGS': ld_flags + flags,
        'OBJECTS': self.deps,
        'EXECUTABLE': self.name })

      try:
        if ld_use_shell: self.cmd = [ 'sh', '-c', ' '.join(self.cmd) ]
      except NameError: pass

      if ld_enabled:
        self.deps.add (os.path.join (lib_dir, libname))



  def set_object (self):
    self.action = 'CC'
    cc_file = self.name[:-len(obj_suffix)] + cpp_suffix
    self.deps = set([ cc_file ])
    flags = copy.copy (gsl_cflags)

    if is_moc (cc_file):
      src_header = cc_file[:-len('_moc'+cpp_suffix)] + h_suffix
      list_headers (src_header)
      file_flags[cc_file] = file_flags[src_header]
    else: self.deps = self.deps.union (list_headers (cc_file))
    self.deps.add (config_file)

    skip = False
    if 'T' in file_flags[cc_file]: flags += thread_cflags
    if 'Q' in file_flags[cc_file]: flags += qt_cflags

    if not skip: 
      self.cmd = fillin (cpp, { 
        'CFLAGS': cpp_flags + flags,
        'OBJECT': self.name,
        'SRC': cc_file })


  def set_moc (self):
    self.action = 'MOC'
    src_file = self.name[:-len('_moc'+cpp_suffix)] + h_suffix
    self.deps = set([ src_file ])
    self.deps = self.deps.union (list_headers (src_file))
    self.cmd = [ 'moc-qt4' ]
    self.cmd += [ src_file, '-o', self.name ]

    
  def set_library (self):
    if not ld_enabled:
      disp ('ERROR: shared library generation is disabled in this configuration')
      exit (1)

    self.action = 'LD'
    self.deps = list_lib_deps()

    self.cmd = fillin (ld_lib, {
      'LDLIB_FLAGS': ld_lib_flags,
      'OBJECTS': [ item for item in self.deps ],
      'LIB': self.name })

    try:
      if ld_use_shell: self.cmd = [ 'sh', '-c', ' '.join(self.cmd) ]
    except NameError: pass


  def set_icon (self):
    self.action = 'WR'
    self.deps = [ icon_dep ]
    self.cmd = windres + [ icon_dep, icon ]


  def display (self):
    print '[' + self.action + '] ' + self.name,
    if not self.timestamp or self.timestamp < self.dep_timestamp: print '[REBUILD]',
    print ':'
    print '  deps: ', ' '.join(self.deps)
    print '  command: ', ' '.join(self.cmd)
    sys.stdout.flush()






###########################################################################
#                         FUNCTION DEFINITIONS
###########################################################################


def default_targets():
  if not os.path.isdir (cmd_dir): 
    print 'ERROR: no "cmd" folder - unable to determine default targets'
    sys.exit (1)
  for entry in os.listdir (cmd_dir):
    if entry.endswith(cpp_suffix):
      targets.append (os.path.join (bin_dir, entry[:-len(cpp_suffix)] + exe_suffix))
  return targets

def is_executable (target):
  return target.split (os.sep)[0] == bin_dir and not is_moc (target)

def is_library (target):
  return target.endswith (lib_suffix) and target.split(os.sep)[-1].startswith (lib_prefix)

def is_object (target):
  return target.endswith (obj_suffix)

def is_moc (target):
  return target.endswith ('_moc' + cpp_suffix)

def is_icon (target):
  return target == icon

def mtime (target):
  if not os.path.exists (target): return None
  return os.stat(target).st_mtime


def fillin (template, keyvalue):
  cmd = []
  for item in template:
    if item in keyvalue: 
      if type(keyvalue[item]) == type (''): cmd += [ keyvalue[item] ]
      else: cmd += keyvalue[item]
    else: cmd += [ item ]
  return cmd



def execute (message, cmd):
  disp (message)
  if verbose: 
    disp (' '.join(cmd))

  try: 
    process = subprocess.Popen (cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    if process.wait() != 0:
      disp ('\nERROR: ' + message + '\n\n' + ' '.join(cmd) + '\n\nfailed with output\n\n' + process.stderr.read())
      return 1
    out = process.stdout.read()
    if len(out):
      disp ('STDOUT: ' + message + '\n' + out)
    out = process.stderr.read()
    if len(out):
      disp ('STDERR: ' + message + '\n' + out)

  except OSError:
    disp (cmd[0] + ': command not found')
    return 1


def print_deps (file, indent=''):
  print indent + file,
  if file in file_flags: 
    if len(file_flags[file]): print '[' + file_flags[file] + ']',
  print ''
  if len(todo[file].deps):
    for entry in todo[file].deps:
      print_deps (entry, indent + '    ')



def list_headers (file):
  global headers, file_flags

  if file not in headers.keys(): 
    headers[file] = set()

    if file not in file_flags: file_flags[file] = ''

    if not os.path.exists (file):
      print 'ERROR: cannot find file "' + file + '"'
      sys.stdout.flush()
      sys.exit(1)
    fd = open (file, 'r')
    for line in fd:
      line = line.strip()
      if line.startswith('#include'):
        line = line[8:].split ('//')[0].split ('/*')[0].strip()
        if line[0] == '"':
          line = line[1:].rstrip('"')
          if line == 'thread/exec.h':
            if 'T' not in file_flags[file]: 
              file_flags[file] += 'T'
          for path in include_paths:
            if os.path.exists (os.path.join (path, line)):
              line = os.path.join (path, line)
              headers[file].add (line)
              [ headers[file].add(entry) for entry in list_headers(line) ]
              break
          else:
            print 'ERROR: cannot find header file \"' + line + '\" (from file \"' + file + '\")'
            sys.stdout.flush()
            sys.exit(1)
        elif line.startswith ('<Q'):
          if 'Q' not in file_flags[file]:
            file_flags[file] += 'Q'
      elif line == 'Q_OBJECT':
        if 'M' not in file_flags[file]:
          file_flags[file] += 'M'

    fd.close()
    for entry in headers[file]:
      for c in file_flags[entry]:
        if c != 'M' and c not in file_flags[file]:
          file_flags[file] += c

  return headers[file]






def list_cmd_deps (file_cc):
  global object_deps, file_flags

  if file_cc not in object_deps.keys():
    object_deps[file_cc] = set([ file_cc[:-len(cpp_suffix)] + obj_suffix ])
    for entry in list_headers (file_cc):
      if os.path.abspath(entry).startswith(os.path.abspath(lib_dir)): continue
      if 'M' in file_flags[entry]: object_deps[file_cc] = object_deps[file_cc].union ([ entry[:-len(h_suffix)] + '_moc' + obj_suffix ])
      entry_cc = entry[:-len(h_suffix)] + cpp_suffix
      if os.path.exists (entry_cc):
        object_deps[file_cc] = object_deps[file_cc].union (list_cmd_deps(entry_cc))
    if file_cc not in file_flags: file_flags[file_cc] = ''
    for entry in headers[file_cc]:
      for c in file_flags[entry]:
        if c != 'M' and c not in file_flags[file_cc]:
          file_flags[file_cc] += c

  return object_deps[file_cc]



def list_lib_deps ():
  deps = set()
  for root, dirs, files in os.walk (lib_dir):
    for file in files:
      if file[0] == '.': continue
      if file.endswith (cpp_suffix):
        deps.add (os.path.join (root, file[:-len(cpp_suffix)] + obj_suffix))

  return (deps)



def build_next (id):
  global todo, lock, stop

  try:
    while not stop:
      current = None
      lock.acquire()
      if len(todo):
        for item in todo.keys():
          if todo[item].currently_being_processed: continue
          unsatisfied_deps = set(todo[item].deps).intersection (todo.keys())
          if not len(unsatisfied_deps):
            todo[item].currently_being_processed = True
            current = item
            break
      else: stop = max (stop, 1)
      lock.release()
  
      if stop: return
      if current == None: 
        time.sleep (0.01)
        continue
  
      target = todo[current]
      if target.execute():
        stop = 2
        return

      lock.acquire()
      del todo[current]
      lock.release()

  except:
    stop = 2
    return
    
  stop = max(stop, 1)



def start_html (fid, title, left, up, home, right):
  fid.write ('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">\n<html>\n<head>\n<meta http-equiv="Content-Type" content="text/html;charset=iso-8859-1">\n')
  fid.write ('<title>MRtrix documentation</title>\n<link rel="stylesheet" href="../stylesheet.css" type="text/css" media=screen>\n</head>\n<body>\n\n')
  fid.write ('<table class=nav>\n<tr>\n<td><a href="' + left + '.html"><img src="../left.png"></a></td>\n')
  fid.write ('<td><a href="' + up + '.html"><img src="../up.png"></a></td>\n')
  fid.write ('<td><a href="' + home + '.html"><img src="../home.png"></a></td>\n')
  fid.write ('<th>' + title + '</th>\n')
  fid.write ('<td><a href="' + right + '.html"><img src="../right.png"></a></td>\n</tr>\n</table>\n')




def gen_command_html_help ():
  binaries = os.listdir (bin_dir)
  binaries.sort()
  description = []

  # generate each program's individual page:
  for n in range (0, len(binaries)):
    print '[DOC] ' + binaries[n]
    sys.stdout.flush()
     
    process = subprocess.Popen ([ binaries[n], '__print_full_usage__' ], stdout=subprocess.PIPE)
    if process.wait() != 0:
      print 'ERROR: unable to execute', binaries[n], ' - aborting'
      sys.stdout.flush()
      return 1
    H = process.stdout.read()
    H = H.splitlines()

    fid = open (os.path.join (doc_dir, 'commands', binaries[n] + '.html'), 'wb')

    if n == 0: prev = 'index'
    else: prev = binaries[n-1]

    if n == len(binaries)-1: next = 'index'
    else: next = binaries[n+1]

    start_html (fid, binaries[n], prev, 'index', '../index', next)
    fid.write ('<h2>Description</h2>\n')

    line = 0
    while not H[line].startswith ('ARGUMENT') and not H[line].startswith('OPTION'):
      if len(description) <= n: description.append (H[line])
      fid.write ('<p>\n' + H[line] + '\n</p>\n')
      line += 1

    arg = []
    opt = []
    while line < len(H)-2:
      if not H[line].startswith ('ARGUMENT') and not H[line].startswith ('OPTION'):
        print 'ERROR: malformed usage for executable "' + binaries[n] + '" - aborting'
        sys.stdout.flush()
        return 1

      if H[line].startswith ('ARGUMENT'):
        S = H[line].split (None, 5)
        A = [ S[1], int(S[2]), int(S[3]), S[4], H[line+1], H[line+2] ]
        if len(opt) > 0: opt[-1].append (A)
        else: arg.append(A)

      elif H[line].startswith ('OPTION'):
        S = H[line].split (None, 4)
        A = [ S[1], int(S[2]), int(S[3]), H[line+1], H[line+2] ]
        opt.append (A)

      line += 3

    fid.write ('<p class=indented><strong>syntax:</strong> &nbsp; &nbsp; ' + binaries[n] + ' [ options ] ')
    for A in arg:
      if A[1] == 0: fid.write ('[ ')
      fid.write (A[0] + ' ')
      if A[2] == 1: fid.write ('[ ' + A[0] + ' ... ')
      if A[1] == 0 or A[2] == 1: fid.write ('] ')
    fid.write ('</p>\n<h2>Arguments</h2>\n<table class=args>\n')


    for A in arg:
      fid.write ('<tr><td><b>' + A[0] + '</b>')
      if A[1] == 0 or A[2] == 1:
        fid.write (' [ ')
        if A[1] == 0: 
          fid.write ('optional')
          if A[2] == 1: fid.write (', ')
        if A[2] == 1: fid.write ('multiples allowed')
        fid.write (' ]')
      fid.write ('</td>\n<td>' + A[5] + '</td></tr>\n')
    fid.write ('</table>\n')

    fid.write ('<h2>Options</h2>\n<table class=args>\n')
    for O in opt:
      fid.write ('<tr><td><b>-' + O[0] + '</b>')
      for A in O[5:]: fid.write ('&nbsp;<i>' + A[0] + '</i>')
      fid.write ('</td>\n<td>' + O[4])
      if len(O) > 5:
        fid.write ('\n<table class=opts>\n')
        for A in O[5:]:
          fid.write ('<tr><td><i>' + A[0] + '</i></td>\n<td>' + A[5] + '</td></tr>\n')
        fid.write ('</table>')
      fid.write ('</td></tr>\n')
    fid.write ('</table>\n')

    fid.write ('</body>\n</html>')

    fid.close()



  fid = open (os.path.join (doc_dir, 'commands', 'index.html'), 'wb')
  start_html (fid, 'list of MRtrix commands', '../faq', '../index', '../index', '../appendix/index')
  fid.write ('<table class=cmdindex width=100%>\n')

  for n in range (0,len(binaries)):
    fid.write ('<tr><td><a href="' + binaries[n] + '.html">' + binaries[n] + '</a></td><td>' + description[n] + '</td></tr>\n')
  fid.write ('</table>\n</body>\n</html>')
  fid.close()




def clean_cmd ():
  files_to_remove = []
  for root, dirs, files in os.walk ('.'):
    for file in files:
      if file[0] == '.': continue
      if file.endswith (obj_suffix) or ( file.startswith (lib_prefix) and file.endswith (lib_suffix) ) or file.endswith ('_moc'+cpp_suffix):
        files_to_remove.append (os.path.join (root, file))
        
  dirs_to_remove = []
  if os.path.isdir (bin_dir):
    for root, dirs, files in os.walk (bin_dir, topdown=False):
      for file in files: files_to_remove.append (os.path.join (root, file))
      for entry in dirs: dirs_to_remove.append (os.path.join (root, entry))
  
  if os.path.isdir (dev_dir):
    print '[RM] development doc'
    sys.stdout.flush()
    for root, dirs, files in os.walk (dev_dir, topdown=False):
      for entry in files:
        os.remove (os.path.join (root, entry))
    for entry in dirs:
        os.rmdir (os.path.join (root, entry))
    os.rmdir (dev_dir)

  if len(files_to_remove):
    print '[RM] ' + ' '.join(files_to_remove)
    sys.stdout.flush()
    for entry in files_to_remove: os.remove (entry)

  if len(dirs_to_remove):
    print '[RM] ' + ' '.join (dirs_to_remove)
    sys.stdout.flush()
    for entry in dirs_to_remove: os.rmdir (entry)



###########################################################################
#                            START OF PROGRAM
###########################################################################



if 'clean' in targets:
  clean_cmd()
  sys.exit (0)

if doc_dir in targets: 
  sys.exit (gen_command_html_help());
    
if dev_dir in targets: 
  sys.exit (execute ('[DOC] development', [ 'doxygen' ]))
    
if len(targets) == 0: targets = default_targets()

if verbose:
  print 'building targets:',
  for entry in targets:
    print entry,
  print ''
  sys.stdout.flush()

if verbose:
  print '' 
  print 'compiling TODO list...'

try: [ Entry(item) for item in targets ]
except Exception, e: 
  print 'ERROR:', e.args[0]
  sys.exit (1)

if dependencies:
  print ''
  print 'Printing dependencies:'
  print ''
  for entry in targets:
    print_deps (entry)
  sys.exit (0)

for item in todo.keys():
  if todo[item].action == 'NA' or (todo[item].timestamp and todo[item].timestamp >= todo[item].dep_timestamp):
    del todo[item]
if verbose: 
  print 'TODO list contains ' + str(len(todo)) + ' items'
  print ''


#for entry in todo.values(): entry.display()

if not len(todo): sys.exit(0)



try: num_processors = os.sysconf('SC_NPROCESSORS_ONLN')
except:
  try: num_processors = int(os.environ['NUMBER_OF_PROCESSORS'])
  except: num_processors = 1
  
if verbose:
  print '' 
  print 'launching ' + str(num_processors) + ' threads'
  print '' 

threads = []
for i in range (1, num_processors):
  t = threading.Thread (target=build_next, args=(i,));
  t.start()
  threads.append (t)

build_next(0)

for t in threads: t.join()

sys.exit (stop > 1)