From 86d2b637bc99e64a334390b6f82b2d636685daa4 Mon Sep 17 00:00:00 2001 From: Igor Gaponenko Date: Fri, 16 Feb 2018 00:30:24 +0000 Subject: [PATCH 1/5] extedended main SConscript to support building executables of C++ code, added a demo application which tests if versions of the Gpoogle Protobuf headers and libraries match --- core/modules/SConscript | 11 +++-- core/modules/tests/SConscript | 38 ++++++++++++++- core/modules/tests/tests_protobuf_version.cc | 49 ++++++++++++++++++++ 3 files changed, 94 insertions(+), 4 deletions(-) create mode 100644 core/modules/tests/tests_protobuf_version.cc diff --git a/core/modules/SConscript b/core/modules/SConscript index 404e961616..3ca18b6e83 100644 --- a/core/modules/SConscript +++ b/core/modules/SConscript @@ -289,15 +289,20 @@ for shlib, shopt in shlibs.items(): state.log.debug('initialized build_data: ' + defEnv.Dump('build_data')) # call all modules' SConscripts or use standard method +# and harvest locally defined build targets (if any) from each module. +build_targets = [] for mod in all_modules: scriptPath = os.path.join(mod, "SConscript") - if env.Glob(scriptPath): # os.access is faulty with VariantDir - SConscript(scriptPath, exports={'env': env, 'standardModule': standardModule}) + if env.Glob(scriptPath): # os.access is faulty with VariantDir + module_build_targets = SConscript(scriptPath, exports={'env': env, 'standardModule': standardModule}) + if module_build_targets: + for t in module_build_targets: + build_targets.append(t) + build_data['install'] += env.Install("$prefix/bin", t) else: standardModule(env, module=mod) # define targets for shared libs -build_targets = [] for shlib, shopt in shlibs.items(): state.log.debug('defining target for ' + shlib) diff --git a/core/modules/tests/SConscript b/core/modules/tests/SConscript index 5f3ff81826..97b8bb6a29 100644 --- a/core/modules/tests/SConscript +++ b/core/modules/tests/SConscript @@ -2,4 +2,40 @@ Import('env') Import('standardModule') -standardModule(env, unit_tests="") +import os.path + +# Harvest special binary products - files starting with the package's name +# followed by underscore: +# +# tests_.cc +# +# Then make a Scons target for each of those to be returned by the calling +# SConscript. Also add the source file into a list of sources to be excluded +# from the shared library product. + +env.Append(LIBPATH=['$build_dir'] + env['LIBPATH']) + +module_build_targets = [] +exclude = [] + +path = "." +for f in env.Glob(os.path.join(path, "tests_*.cc"), source=True, strings=True): + source = os.path.basename(str(f)) + target = source[:-3] + module_build_targets += env.Program( + target=target, + source=[source], + LIBS =["qserv_common", + "util", + "protobuf", + "log", + "log4cxx"]) + exclude.append(f) + +# Initiate the standard sequence of actions for this module by excluding +# the above discovered bibary sources + +standardModule(env, exclude=exclude, unit_tests="") + +Return('module_build_targets') +s diff --git a/core/modules/tests/tests_protobuf_version.cc b/core/modules/tests/tests_protobuf_version.cc new file mode 100644 index 0000000000..d82270ccb9 --- /dev/null +++ b/core/modules/tests/tests_protobuf_version.cc @@ -0,0 +1,49 @@ +/* + * LSST Data Management System + * Copyright 2009-2018 AURA/LSST. + * + * This product includes software developed by the + * LSST Project (http://www.lsst.org/). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the LSST License Statement and + * the GNU General Public License along with this program. If not, + * see . + */ + +// System headers + +// Third-party headers + +// LSST headers +#include "lsst/log/Log.h" + +// Qserv headers +#include "proto/worker.pb.h" + +namespace { +LOG_LOGGER _log = LOG_GET("lsst.qserv.tests.tests_protobuf_version"); +} + +int main (int arc, char const* argv[]) { + + LOGS(_log, LOG_LVL_INFO, "testing a version of the Protobuf library"); + + // Verify that the version of the library that we linked against is + // compatible with the version of the headers we compiled against. + + GOOGLE_PROTOBUF_VERIFY_VERSION; + + LOGS(_log, LOG_LVL_INFO, "success"); + + return 0; +} From e15005e4c9968df2c877ac77c028819238461537 Mon Sep 17 00:00:00 2001 From: Igor Gaponenko Date: Fri, 16 Feb 2018 04:17:50 +0000 Subject: [PATCH 2/5] improved version of the build tool which moves the common and boilerplate code into function standardModule. This should simplify module-level SConscript file, and it will also make them more robust --- core/modules/SConscript | 52 ++++++++++++++++++++++++++--------- core/modules/tests/SConscript | 25 +++-------------- 2 files changed, 43 insertions(+), 34 deletions(-) diff --git a/core/modules/SConscript b/core/modules/SConscript index 3ca18b6e83..0db32d9921 100644 --- a/core/modules/SConscript +++ b/core/modules/SConscript @@ -70,7 +70,7 @@ import state Import('env') -def standardModule(env, exclude=None, module=None, test_libs=None, unit_tests=None): +def standardModule(env, exclude=None, module=None, test_libs=None, unit_tests=None, bin_cc_targets=None): """ Implementation of SConscript logic for "standard" module. @@ -95,6 +95,14 @@ def standardModule(env, exclude=None, module=None, test_libs=None, unit_tests=No then use this parameter to pass list (or space-separated string) of application names. Can be empty string or list to not run any unit tests. + @param bin_cc_files: a dictionary defining whgich binary products (not + the unit tests should be built out of C/C++ files. Each key + of the dictionary is the name of a C/C++ file (with function + 'main'). This file will be excluded from the unit tests and + from a list of *.cc files which are complied into a shared + library of the module. And the corresponding value of + the dictionary is a list of libraries which are required to + build the binary. """ build_data = DefaultEnvironment()['build_data'] @@ -104,13 +112,23 @@ def standardModule(env, exclude=None, module=None, test_libs=None, unit_tests=No module = os.path.basename(os.getcwd()) state.log.debug('standardModule: module = %s path = %s' % (module, path)) + # using two optional lists provided by the nmodule to build names of + # *.cc files to be excluded from being used to build module's + # library or unit tests + exclude_cc_files = [] + if exclude is not None: + exclude_cc_files = exclude + if bin_cc_targets is not None: + for f in bin_cc_targets: exclude_cc_files.append(f) + # find all *.cc files and split then in two groups: files for libraries - # and files for test apps - cc_files = set(env.Glob(os.path.join(path, "*.cc"), source=True, strings=True, exclude=exclude)) + # and files for test apps. + cc_files = set(env.Glob(os.path.join(path, "*.cc"), source=True, strings=True, exclude=exclude_cc_files)) cc_tests = set(fname for fname in cc_files if os.path.basename(fname).startswith("test")) # In Python3 order of set iteration is undefined, do manual sorting cc_objects = sorted(cc_files - cc_tests) cc_tests = sorted(cc_tests) + state.log.debug('standardModule: exclude_cc_files = ' + str(exclude_cc_files)) state.log.debug('standardModule: cc_objects = ' + str(cc_objects)) state.log.debug('standardModule: cc_tests = ' + str(cc_tests)) @@ -163,6 +181,18 @@ def standardModule(env, exclude=None, module=None, test_libs=None, unit_tests=No py_installed = env.Install("$prefix/bin", py_scripts) build_data['install'] += py_installed + # inject binary targets build of the optionally provided C/C++ files + if bin_cc_targets is not None: + for f,dependencies in bin_cc_targets.items(): + source = os.path.basename(str(f)) + target = source[:-3] + p = env.Program( + target=target, + source=[source], + LIBS=dependencies, + LIBPATH=['$build_dir'] + env['LIBPATH']) + build_data['build_targets'] += p + build_data['install'] += env.Install("$prefix/bin", p) # ================== SConscript logic start here ==================== @@ -274,6 +304,7 @@ build_data['shlibs'] = shlibs.copy() build_data['tests'] = [] build_data['unit_tests'] = [] build_data['module_lib'] = defaultdict(lambda: None) +build_data['build_targets'] = [] # store in env so that modules can use it defEnv = DefaultEnvironment() @@ -290,15 +321,10 @@ state.log.debug('initialized build_data: ' + defEnv.Dump('build_data')) # call all modules' SConscripts or use standard method # and harvest locally defined build targets (if any) from each module. -build_targets = [] for mod in all_modules: scriptPath = os.path.join(mod, "SConscript") if env.Glob(scriptPath): # os.access is faulty with VariantDir - module_build_targets = SConscript(scriptPath, exports={'env': env, 'standardModule': standardModule}) - if module_build_targets: - for t in module_build_targets: - build_targets.append(t) - build_data['install'] += env.Install("$prefix/bin", t) + SConscript(scriptPath, exports={'env': env, 'standardModule': standardModule}) else: standardModule(env, module=mod) @@ -324,7 +350,7 @@ for shlib, shopt in shlibs.items(): pfx = shopt.get('SHLIBPREFIX', env['SHLIBPREFIX']) out = env.SharedLibrary(shlib, objects, LIBS=Split(shopt['libs']), SHLIBPREFIX=pfx, LIBPATH=libPath) - build_targets += out + build_data['build_targets'] += out # install target instDir = shopt.get('instDir', 'lib') @@ -346,11 +372,11 @@ if utests: # special builder that checks that unit tests were successful utests = env.UnitTestCheck('unit-test-check-tag', utests) -build_targets += build_data['tests'] +build_data['build_targets'] += build_data['tests'] # define/extend all aliases -state.log.debug("build targets: %s" % map(str, build_targets)) -env.Alias("build", build_targets) +state.log.debug("build targets: %s" % map(str, build_data['build_targets'])) +env.Alias("build", build_data['build_targets']) state.log.debug("install targets: %s" % map(str, build_data['install'])) env.Alias("install-notest", build_data['install']) if utests: diff --git a/core/modules/tests/SConscript b/core/modules/tests/SConscript index 97b8bb6a29..cbf3ab3602 100644 --- a/core/modules/tests/SConscript +++ b/core/modules/tests/SConscript @@ -13,29 +13,12 @@ import os.path # SConscript. Also add the source file into a list of sources to be excluded # from the shared library product. -env.Append(LIBPATH=['$build_dir'] + env['LIBPATH']) - -module_build_targets = [] -exclude = [] - +bin_cc_targets = {} path = "." for f in env.Glob(os.path.join(path, "tests_*.cc"), source=True, strings=True): - source = os.path.basename(str(f)) - target = source[:-3] - module_build_targets += env.Program( - target=target, - source=[source], - LIBS =["qserv_common", - "util", - "protobuf", - "log", - "log4cxx"]) - exclude.append(f) + bin_cc_targets[f] = ["qserv_common","util","protobuf","log","log4cxx"] # Initiate the standard sequence of actions for this module by excluding -# the above discovered bibary sources - -standardModule(env, exclude=exclude, unit_tests="") +# the above discovered binary sources -Return('module_build_targets') -s +standardModule(env, bin_cc_targets=bin_cc_targets, unit_tests="") From cd5b0db04080c5fcc95e393fc21f6d767559322f Mon Sep 17 00:00:00 2001 From: Igor Gaponenko Date: Fri, 16 Feb 2018 04:18:57 +0000 Subject: [PATCH 3/5] fixed a small typo --- core/modules/SConscript | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/modules/SConscript b/core/modules/SConscript index 0db32d9921..74e8d34f5a 100644 --- a/core/modules/SConscript +++ b/core/modules/SConscript @@ -95,7 +95,7 @@ def standardModule(env, exclude=None, module=None, test_libs=None, unit_tests=No then use this parameter to pass list (or space-separated string) of application names. Can be empty string or list to not run any unit tests. - @param bin_cc_files: a dictionary defining whgich binary products (not + @param bin_cc_targets: a dictionary defining whgich binary products (not the unit tests should be built out of C/C++ files. Each key of the dictionary is the name of a C/C++ file (with function 'main'). This file will be excluded from the unit tests and From ba1bfb2adef8b78bf1bbbfc6d5b7a78cd177f46c Mon Sep 17 00:00:00 2001 From: Igor Gaponenko Date: Fri, 16 Feb 2018 04:25:07 +0000 Subject: [PATCH 4/5] fixed minor typos and indentetions --- core/modules/SConscript | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/core/modules/SConscript b/core/modules/SConscript index 74e8d34f5a..fadf7db23c 100644 --- a/core/modules/SConscript +++ b/core/modules/SConscript @@ -122,7 +122,7 @@ def standardModule(env, exclude=None, module=None, test_libs=None, unit_tests=No for f in bin_cc_targets: exclude_cc_files.append(f) # find all *.cc files and split then in two groups: files for libraries - # and files for test apps. + # and files for test apps cc_files = set(env.Glob(os.path.join(path, "*.cc"), source=True, strings=True, exclude=exclude_cc_files)) cc_tests = set(fname for fname in cc_files if os.path.basename(fname).startswith("test")) # In Python3 order of set iteration is undefined, do manual sorting @@ -181,16 +181,16 @@ def standardModule(env, exclude=None, module=None, test_libs=None, unit_tests=No py_installed = env.Install("$prefix/bin", py_scripts) build_data['install'] += py_installed - # inject binary targets build of the optionally provided C/C++ files + # inject binary targets built of the optionally provided C/C++ files if bin_cc_targets is not None: for f,dependencies in bin_cc_targets.items(): source = os.path.basename(str(f)) target = source[:-3] p = env.Program( - target=target, - source=[source], - LIBS=dependencies, - LIBPATH=['$build_dir'] + env['LIBPATH']) + target=target, + source=[source], + LIBS=dependencies, + LIBPATH=['$build_dir'] + env['LIBPATH']) build_data['build_targets'] += p build_data['install'] += env.Install("$prefix/bin", p) @@ -320,10 +320,9 @@ for shlib, shopt in shlibs.items(): state.log.debug('initialized build_data: ' + defEnv.Dump('build_data')) # call all modules' SConscripts or use standard method -# and harvest locally defined build targets (if any) from each module. for mod in all_modules: scriptPath = os.path.join(mod, "SConscript") - if env.Glob(scriptPath): # os.access is faulty with VariantDir + if env.Glob(scriptPath): # os.access is faulty with VariantDir SConscript(scriptPath, exports={'env': env, 'standardModule': standardModule}) else: standardModule(env, module=mod) From 6860c8431cfecfad97db163097f9f5fa8f26ad54 Mon Sep 17 00:00:00 2001 From: Igor Gaponenko Date: Fri, 16 Feb 2018 08:04:58 +0000 Subject: [PATCH 5/5] made changes as requested by the reviewer --- core/modules/SConscript | 28 ++++++++----------- core/modules/tests/SConscript | 8 +++--- ...on.cc => qserv-protobuf-assert-version.cc} | 0 3 files changed, 15 insertions(+), 21 deletions(-) rename core/modules/tests/{tests_protobuf_version.cc => qserv-protobuf-assert-version.cc} (100%) diff --git a/core/modules/SConscript b/core/modules/SConscript index fadf7db23c..3414ce2c9d 100644 --- a/core/modules/SConscript +++ b/core/modules/SConscript @@ -70,7 +70,7 @@ import state Import('env') -def standardModule(env, exclude=None, module=None, test_libs=None, unit_tests=None, bin_cc_targets=None): +def standardModule(env, exclude=None, module=None, test_libs=None, unit_tests=None, bin_cc_files=None): """ Implementation of SConscript logic for "standard" module. @@ -95,8 +95,8 @@ def standardModule(env, exclude=None, module=None, test_libs=None, unit_tests=No then use this parameter to pass list (or space-separated string) of application names. Can be empty string or list to not run any unit tests. - @param bin_cc_targets: a dictionary defining whgich binary products (not - the unit tests should be built out of C/C++ files. Each key + @param bin_cc_files: a dictionary defining which binary products (not + the unit tests) should be built out of C/C++ files. Each key of the dictionary is the name of a C/C++ file (with function 'main'). This file will be excluded from the unit tests and from a list of *.cc files which are complied into a shared @@ -117,9 +117,9 @@ def standardModule(env, exclude=None, module=None, test_libs=None, unit_tests=No # library or unit tests exclude_cc_files = [] if exclude is not None: - exclude_cc_files = exclude - if bin_cc_targets is not None: - for f in bin_cc_targets: exclude_cc_files.append(f) + if isinstance(exclude, str): exclude_cc_files += [exclude,] + else: exclude_cc_files += exclude + if bin_cc_files is not None: exclude_cc_files += bin_cc_files.keys() # find all *.cc files and split then in two groups: files for libraries # and files for test apps @@ -182,17 +182,11 @@ def standardModule(env, exclude=None, module=None, test_libs=None, unit_tests=No build_data['install'] += py_installed # inject binary targets built of the optionally provided C/C++ files - if bin_cc_targets is not None: - for f,dependencies in bin_cc_targets.items(): - source = os.path.basename(str(f)) - target = source[:-3] - p = env.Program( - target=target, - source=[source], - LIBS=dependencies, - LIBPATH=['$build_dir'] + env['LIBPATH']) - build_data['build_targets'] += p - build_data['install'] += env.Install("$prefix/bin", p) + if bin_cc_files is not None: + for f,dependencies in bin_cc_files.items(): + p = env.Program(f, LIBS=dependencies, LIBPATH=['$build_dir'] + env['LIBPATH']) + build_data['build_targets'] += p + build_data['install'] += env.Install("$prefix/bin", p) # ================== SConscript logic start here ==================== diff --git a/core/modules/tests/SConscript b/core/modules/tests/SConscript index cbf3ab3602..66fe89e4cb 100644 --- a/core/modules/tests/SConscript +++ b/core/modules/tests/SConscript @@ -13,12 +13,12 @@ import os.path # SConscript. Also add the source file into a list of sources to be excluded # from the shared library product. -bin_cc_targets = {} +bin_cc_files = {} path = "." -for f in env.Glob(os.path.join(path, "tests_*.cc"), source=True, strings=True): - bin_cc_targets[f] = ["qserv_common","util","protobuf","log","log4cxx"] +for f in env.Glob(os.path.join(path, "qserv-*.cc"), source=True, strings=True): + bin_cc_files[f] = ["qserv_common","util","protobuf","log","log4cxx"] # Initiate the standard sequence of actions for this module by excluding # the above discovered binary sources -standardModule(env, bin_cc_targets=bin_cc_targets, unit_tests="") +standardModule(env, bin_cc_files=bin_cc_files, unit_tests="") diff --git a/core/modules/tests/tests_protobuf_version.cc b/core/modules/tests/qserv-protobuf-assert-version.cc similarity index 100% rename from core/modules/tests/tests_protobuf_version.cc rename to core/modules/tests/qserv-protobuf-assert-version.cc