Miscellaneous fixes, python packaging and python API improvements (#327)

* fix(swig) Make enums and constants available in python

* fix(python) edata is regular pointer, not unique_ptr here

* refactor(cmake)

* fix(python) Fix linking order. Fix C++ standard.

* feature(python) Add verbosity switch for model compilation

* feature(python) Add verbosity switch for sbml2amici

* fix(python) Make swig aware of enums in time to generate appropriate mappers

* doc(python) Extend python API steadystate example. Change to notebook.

* fix(python) Check edata type. Here it can be regular or unique pointer

* fix(python) Check edata type. Here it can be regular or unique pointer

* doc(python) Cleanup notebook

* doc(python) Add finite differences sensitivity check to example

* feature(python) Allow blas selection for amici base package installation via environment variables #320

* doc(python) Add model info to example

* Fix(python) Fix negative log likelihood

* Feature(python) Expose std::vector<amici::AMICI_parameter_scaling> as ParameterScalingVector

* doc(core) Fix dimension order

* ci(python) Run gradient check for example models (don't fail test yet)

* ci(travis) Install scipy

* build(cmake) CMake cleanup

* ci(travis) Test creating and installing sdist on ci server

* fix(python) Increase chances of finding swig on OSX #320

* ci(python) Don't use pyenv pip which is broken by recent update

  File '/opt/pyenv/versions/3.6/bin/pip3', line 7, in <module>
    from pip import main
ImportError: cannot import name 'main'

* fix(python) Catch hdf5.cpp not found problem #320

* doc(doxygen) Ignore autogenerated file

* fix(python) Set sensitivity flag for dydp (thx FFroehlich)

* feature(python) Exception on package setup failure

* fix(python) Fix sx summation. Fixes sx gradient check (thx FFroehlich)

* fix(python) fix sy computation in python generated models

* Fix typo, rewrap models, remove notebook test code

.travis.yml

@@ -58,13 +58,17 @@ install:
     - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then pyenv versions; fi
     - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then pyenv shell 2.7 3.6; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export PATH=/Users/travis/Library/Python/3.6/bin:$PATH; fi
-    - pip3 install --user --upgrade pip setuptools wheel pkgconfig doxypypy coverage
+    - pip3 install --user --upgrade pip setuptools wheel pkgconfig doxypypy coverage scipy
     - ./scripts/buildSuiteSparse.sh
     - ./scripts/buildSundials.sh
     - ./scripts/buildCpputest.sh
     - ./scripts/buildAmici.sh
 
 script:  
+    - cd $BASE_DIR/python/sdist
+    - python3 setup.py sdist --dist-dir=$BASE_DIR/build/python/
+    - python3 -m pip install $BASE_DIR/build/python/amici-*.tar.gz --verbose
+    - cd $BASE_DIR
     - ./scripts/run-cpputest.sh
     - ./scripts/run-cppcheck.sh
     - cd $BASE_DIR/build && make python-tests && cd $BASE_DIR

include/amici/rdata.h

@@ -83,7 +83,7 @@ class ReturnData {
     /** state (dimension: nt x nx, row-major) */
     std::vector<realtype> x;
 
-    /** parameter derivative of state (dimension: nt x nx x nplist,
+    /** parameter derivative of state (dimension: nt x nplist x nx,
      * row-major) */
     std::vector<realtype> sx;
 
@@ -93,12 +93,12 @@ class ReturnData {
     /** observable standard deviation (dimension: nt x ny, row-major) */
     std::vector<realtype> sigmay;
 
-    /** parameter derivative of observable (dimension: nt x ny x nplist,
+    /** parameter derivative of observable (dimension: nt x nplist x ny,
      * row-major) */
     std::vector<realtype> sy;
 
     /** parameter derivative of observable standard deviation (dimension: nt x
-     * ny x nplist, row-major) */
+     * nplist x ny, row-major) */
     std::vector<realtype> ssigmay;
 
     /** observable (dimension: nt x ny, row-major) */

matlab/mtoc/config/Doxyfile.template

@@ -697,6 +697,7 @@ EXCLUDE_PATTERNS       = "_SourceDir_/models/*" \
                          "_SourceDir_/matlab/auxiliary/*" \
                          "_SourceDir_/matlab/SBMLImporter/*" \
                          "_SourceDir_/python/test/*" \
+                         "_SourceDir_/python/sdist/*" \
                          "_SourceDir_/build/*" \ 
                          "_SourceDir_/build_xcode/*" \
                          "_SourceDir_/swig/*"

models/model_dirac/model_dirac.h

@@ -1,6 +1,6 @@
 #ifndef _amici_model_dirac_h
 #define _amici_model_dirac_h
-/* Generated by amiwrap (R2017b) 2632f34b3b684c381e19069e73b4b1277492fcf1 */
+/* Generated by amiwrap (R2017b) 53e279f979516a66d9b07a42c171c4fa35b6f26d */
 #include <cmath>
 #include <memory>
 #include "amici/defines.h"
@@ -62,7 +62,7 @@ class Model_model_dirac : public amici::Model_ODE {
 
     virtual amici::Model* clone() const override { return new Model_model_dirac(*this); };
 
-    const  char* getAmiciVersion() const { return "2632f34b3b684c381e19069e73b4b1277492fcf1"; };
+    const  char* getAmiciVersion() const { return "53e279f979516a66d9b07a42c171c4fa35b6f26d"; };
 
     virtual void fJ(realtype *J, const realtype t, const realtype *x, const double *p, const double *k, const realtype *h, const realtype *w, const realtype *dwdx) override {
         J_model_dirac(J, t, x, p, k, h, w, dwdx);

models/model_events/model_events.h

@@ -1,6 +1,6 @@
 #ifndef _amici_model_events_h
 #define _amici_model_events_h
-/* Generated by amiwrap (R2017b) 2632f34b3b684c381e19069e73b4b1277492fcf1 */
+/* Generated by amiwrap (R2017b) 53e279f979516a66d9b07a42c171c4fa35b6f26d */
 #include <cmath>
 #include <memory>
 #include "amici/defines.h"
@@ -76,7 +76,7 @@ class Model_model_events : public amici::Model_ODE {
 
     virtual amici::Model* clone() const override { return new Model_model_events(*this); };
 
-    const  char* getAmiciVersion() const { return "2632f34b3b684c381e19069e73b4b1277492fcf1"; };
+    const  char* getAmiciVersion() const { return "53e279f979516a66d9b07a42c171c4fa35b6f26d"; };
 
     virtual void fJ(realtype *J, const realtype t, const realtype *x, const double *p, const double *k, const realtype *h, const realtype *w, const realtype *dwdx) override {
         J_model_events(J, t, x, p, k, h, w, dwdx);

models/model_jakstat_adjoint/model_jakstat_adjoint.h

@@ -1,6 +1,6 @@
 #ifndef _amici_model_jakstat_adjoint_h
 #define _amici_model_jakstat_adjoint_h
-/* Generated by amiwrap (R2017b) 2632f34b3b684c381e19069e73b4b1277492fcf1 */
+/* Generated by amiwrap (R2017b) 53e279f979516a66d9b07a42c171c4fa35b6f26d */
 #include <cmath>
 #include <memory>
 #include "amici/defines.h"
@@ -65,7 +65,7 @@ class Model_model_jakstat_adjoint : public amici::Model_ODE {
 
     virtual amici::Model* clone() const override { return new Model_model_jakstat_adjoint(*this); };
 
-    const  char* getAmiciVersion() const { return "2632f34b3b684c381e19069e73b4b1277492fcf1"; };
+    const  char* getAmiciVersion() const { return "53e279f979516a66d9b07a42c171c4fa35b6f26d"; };
 
     virtual void fJ(realtype *J, const realtype t, const realtype *x, const double *p, const double *k, const realtype *h, const realtype *w, const realtype *dwdx) override {
         J_model_jakstat_adjoint(J, t, x, p, k, h, w, dwdx);

models/model_jakstat_adjoint_o2/model_jakstat_adjoint_o2.h

@@ -1,6 +1,6 @@
 #ifndef _amici_model_jakstat_adjoint_o2_h
 #define _amici_model_jakstat_adjoint_o2_h
-/* Generated by amiwrap (R2017b) 2632f34b3b684c381e19069e73b4b1277492fcf1 */
+/* Generated by amiwrap (R2017b) 53e279f979516a66d9b07a42c171c4fa35b6f26d */
 #include <cmath>
 #include <memory>
 #include "amici/defines.h"
@@ -65,7 +65,7 @@ class Model_model_jakstat_adjoint_o2 : public amici::Model_ODE {
 
     virtual amici::Model* clone() const override { return new Model_model_jakstat_adjoint_o2(*this); };
 
-    const  char* getAmiciVersion() const { return "2632f34b3b684c381e19069e73b4b1277492fcf1"; };
+    const  char* getAmiciVersion() const { return "53e279f979516a66d9b07a42c171c4fa35b6f26d"; };
 
     virtual void fJ(realtype *J, const realtype t, const realtype *x, const double *p, const double *k, const realtype *h, const realtype *w, const realtype *dwdx) override {
         J_model_jakstat_adjoint_o2(J, t, x, p, k, h, w, dwdx);

models/model_nested_events/model_nested_events.h

@@ -1,6 +1,6 @@
 #ifndef _amici_model_nested_events_h
 #define _amici_model_nested_events_h
-/* Generated by amiwrap (R2017b) 2632f34b3b684c381e19069e73b4b1277492fcf1 */
+/* Generated by amiwrap (R2017b) 53e279f979516a66d9b07a42c171c4fa35b6f26d */
 #include <cmath>
 #include <memory>
 #include "amici/defines.h"
@@ -65,7 +65,7 @@ class Model_model_nested_events : public amici::Model_ODE {
 
     virtual amici::Model* clone() const override { return new Model_model_nested_events(*this); };
 
-    const  char* getAmiciVersion() const { return "2632f34b3b684c381e19069e73b4b1277492fcf1"; };
+    const  char* getAmiciVersion() const { return "53e279f979516a66d9b07a42c171c4fa35b6f26d"; };
 
     virtual void fJ(realtype *J, const realtype t, const realtype *x, const double *p, const double *k, const realtype *h, const realtype *w, const realtype *dwdx) override {
         J_model_nested_events(J, t, x, p, k, h, w, dwdx);

models/model_neuron/model_neuron.h

@@ -1,6 +1,6 @@
 #ifndef _amici_model_neuron_h
 #define _amici_model_neuron_h
-/* Generated by amiwrap (R2017b) 2632f34b3b684c381e19069e73b4b1277492fcf1 */
+/* Generated by amiwrap (R2017b) 53e279f979516a66d9b07a42c171c4fa35b6f26d */
 #include <cmath>
 #include <memory>
 #include "amici/defines.h"
@@ -79,7 +79,7 @@ class Model_model_neuron : public amici::Model_ODE {
 
     virtual amici::Model* clone() const override { return new Model_model_neuron(*this); };
 
-    const  char* getAmiciVersion() const { return "2632f34b3b684c381e19069e73b4b1277492fcf1"; };
+    const  char* getAmiciVersion() const { return "53e279f979516a66d9b07a42c171c4fa35b6f26d"; };
 
     virtual void fJ(realtype *J, const realtype t, const realtype *x, const double *p, const double *k, const realtype *h, const realtype *w, const realtype *dwdx) override {
         J_model_neuron(J, t, x, p, k, h, w, dwdx);

models/model_neuron_o2/model_neuron_o2.h

@@ -1,6 +1,6 @@
 #ifndef _amici_model_neuron_o2_h
 #define _amici_model_neuron_o2_h
-/* Generated by amiwrap (R2017b) 2632f34b3b684c381e19069e73b4b1277492fcf1 */
+/* Generated by amiwrap (R2017b) 53e279f979516a66d9b07a42c171c4fa35b6f26d */
 #include <cmath>
 #include <memory>
 #include "amici/defines.h"
@@ -81,7 +81,7 @@ class Model_model_neuron_o2 : public amici::Model_ODE {
 
     virtual amici::Model* clone() const override { return new Model_model_neuron_o2(*this); };
 
-    const  char* getAmiciVersion() const { return "2632f34b3b684c381e19069e73b4b1277492fcf1"; };
+    const  char* getAmiciVersion() const { return "53e279f979516a66d9b07a42c171c4fa35b6f26d"; };
 
     virtual void fJ(realtype *J, const realtype t, const realtype *x, const double *p, const double *k, const realtype *h, const realtype *w, const realtype *dwdx) override {
         J_model_neuron_o2(J, t, x, p, k, h, w, dwdx);

models/model_robertson/model_robertson.h

@@ -1,6 +1,6 @@
 #ifndef _amici_model_robertson_h
 #define _amici_model_robertson_h
-/* Generated by amiwrap (R2017b) 2632f34b3b684c381e19069e73b4b1277492fcf1 */
+/* Generated by amiwrap (R2017b) 53e279f979516a66d9b07a42c171c4fa35b6f26d */
 #include <cmath>
 #include <memory>
 #include "amici/defines.h"
@@ -63,7 +63,7 @@ class Model_model_robertson : public amici::Model_DAE {
 
     virtual amici::Model* clone() const override { return new Model_model_robertson(*this); };
 
-    const  char* getAmiciVersion() const { return "2632f34b3b684c381e19069e73b4b1277492fcf1"; };
+    const  char* getAmiciVersion() const { return "53e279f979516a66d9b07a42c171c4fa35b6f26d"; };
 
     virtual void fJ(realtype *J, const realtype t, const realtype *x, const double *p, const double *k, const realtype *h, const realtype cj, const realtype *dx, const realtype *w, const realtype *dwdx) override {
         J_model_robertson(J, t, x, p, k, h, cj, dx, w, dwdx);

models/model_steadystate/model_steadystate.h

@@ -1,6 +1,6 @@
 #ifndef _amici_model_steadystate_h
 #define _amici_model_steadystate_h
-/* Generated by amiwrap (R2017b) 2632f34b3b684c381e19069e73b4b1277492fcf1 */
+/* Generated by amiwrap (R2017b) 53e279f979516a66d9b07a42c171c4fa35b6f26d */
 #include <cmath>
 #include <memory>
 #include "amici/defines.h"
@@ -62,7 +62,7 @@ class Model_model_steadystate : public amici::Model_ODE {
 
     virtual amici::Model* clone() const override { return new Model_model_steadystate(*this); };
 
-    const  char* getAmiciVersion() const { return "2632f34b3b684c381e19069e73b4b1277492fcf1"; };
+    const  char* getAmiciVersion() const { return "53e279f979516a66d9b07a42c171c4fa35b6f26d"; };
 
     virtual void fJ(realtype *J, const realtype t, const realtype *x, const double *p, const double *k, const realtype *h, const realtype *w, const realtype *dwdx) override {
         J_model_steadystate(J, t, x, p, k, h, w, dwdx);

python/CMakeLists.txt

@@ -2,11 +2,11 @@ find_package(PythonInterp 3.6 REQUIRED)
 
 # Configure setup.py and copy to output directory
 set(AMICI_VERSION "@AMICI_VERSION@") # to be replaced later
-SET(SETUP_PY_IN ${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in)
-SET(SETUP_PY_OUT ${CMAKE_CURRENT_BINARY_DIR}/setup.py)
-CONFIGURE_FILE(${SETUP_PY_IN} ${SETUP_PY_OUT})
+set(SETUP_PY_IN ${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in)
+set(SETUP_PY_OUT ${CMAKE_CURRENT_BINARY_DIR}/setup.py)
+configure_file(${SETUP_PY_IN} ${SETUP_PY_OUT})
 
-CONFIGURE_FILE(${PROJECT_SOURCE_DIR}/python/amici/setup.template.py ${CMAKE_CURRENT_BINARY_DIR}/setup.template.py)
+configure_file(${PROJECT_SOURCE_DIR}/python/amici/setup.template.py ${CMAKE_CURRENT_BINARY_DIR}/setup.template.py)
 
 # Copy further amici-python-module files to binary_dir
 add_custom_target(python-module-files
@@ -53,7 +53,9 @@ add_custom_target(python-wheel
     COMMAND ${PYTHON_EXECUTABLE} setup.py bdist_wheel --dist-dir=${CMAKE_CURRENT_BINARY_DIR}
 )
 
+
 add_custom_command(
     OUTPUT always_rebuild
     COMMAND cmake -E echo
     )
+

python/amici/__init__.py

@@ -69,8 +69,9 @@ def runAmiciSimulation(model, solver, edata=None):
     Raises:
         
     """
-    if edata:
+    if edata and edata.__class__.__name__ == 'ExpDataPtr':
         edata = edata.get()
+
     rdata = amici.runAmiciSimulation(solver.get(), edata, model.get())
     return rdataToNumPyArrays(rdata)
 

python/amici/sbml_import.py

@@ -133,7 +133,9 @@ def __init__(self, SBMLFile):
                              ' const realtype *k, const realtype *h)'},
             'dydp': {
                 'signature': '(double *dydp, const realtype t, const realtype *x, const realtype *p,'
-                             ' const realtype *k, const realtype *h)'},
+                             ' const realtype *k, const realtype *h, const int ip)',
+                'sensitivity': True},
+
             'qBdot': {
                 'signature': '(realtype *qBdot, const int ip, const realtype t, const realtype *x, const realtype *p,'
                              ' const realtype *k, const realtype *h, const realtype *xB, const realtype *w,'
@@ -222,7 +224,7 @@ def loadSBMLFile(self, SBMLFile):
         self.sbml = self.sbml_doc.getModel()
 
 
-    def sbml2amici(self, modelName, output_dir=None, observables={}, constantParameters=[], sigmas={}):
+    def sbml2amici(self, modelName, output_dir=None, observables={}, constantParameters=[], sigmas={}, verbose=False):
         """Generate AMICI C++ files for the model provided to the constructor.
         
         Arguments:
@@ -231,6 +233,7 @@ def sbml2amici(self, modelName, output_dir=None, observables={}, constantParamet
             observables: dictionary(observableName:formulaString) to be added to the model
             sigmas: dictionary(observableName: sigma value or (existing) parameter name)
             constantParameters: list of SBML Ids identifying constant parameters
+            verbose: more verbose output if True
         Returns:
 
         Raises:
@@ -243,7 +246,7 @@ def sbml2amici(self, modelName, output_dir=None, observables={}, constantParamet
         self.computeModelEquations(observables, sigmas)
         self.prepareModelFolder()
         self.generateCCode()
-        self.compileCCode()
+        self.compileCCode(verbose)
 
 
     def setName(self, modelName):
@@ -757,7 +760,7 @@ def computeModelEquationsObjectiveFunction(self, observables={}, sigmas={}):
 
         self.symbols['my']['sym'] = sp.DenseMatrix([sp.sympify('m' + str(symbol)) for symbol in observableSyms])
 
-        loglikelihoodString = lambda strSymbol: '0.5*sqrt(2*pi*sigma{symbol}**2) + 0.5*(({symbol}-m{symbol})/sigma{symbol})**2'.format(symbol=strSymbol)
+        loglikelihoodString = lambda strSymbol: '0.5*log(2*pi*sigma{symbol}**2) + 0.5*(({symbol}-m{symbol})/sigma{symbol})**2'.format(symbol=strSymbol)
         self.functions['Jy']['sym'] = sp.DenseMatrix([sp.sympify(loglikelihoodString(str(symbol))) for symbol in observableSyms])
         self.functions['dJydy']['sym'] = self.functions['Jy']['sym'].jacobian(observableSyms)
         self.functions['dJydsigma']['sym'] = self.functions['Jy']['sym'].jacobian(sigmaYSyms)
@@ -782,7 +785,7 @@ def computeModelEquationsSensitivitesCore(self):
         self.functions['dydp']['sym'] = self.functions['y']['sym']\
                                                 .jacobian(self.symbols['parameter']['sym'])
         self.functions['dydx']['sym'] = self.functions['y']['sym']\
-                                                .jacobian(self.symbols['species']['sym'])
+                                                .jacobian(self.symbols['species']['sym']).transpose()
 
         self.functions['dwdp']['sym'] = self.fluxVector.jacobian(self.symbols['parameter']['sym'])
 
@@ -882,26 +885,43 @@ def generateCCode(self):
                     os.path.join(self.modelPath, 'main.cpp'))
 
 
-    def compileCCode(self):
+    def compileCCode(self, verbose=False):
         """Compile the generated model code
         
         Arguments:
-
+            verbose: Make model compilation verbose
         Returns:
 
         Raises:
 
         """
-
+
+        # setup.py assumes it is run from within the model directory
         moduleDir = self.modelPath
-        oldCwd = os.getcwd()
-        os.chdir(moduleDir) # setup.py assumes it is run from within the model dir
-        from distutils.core import run_setup
-        run_setup('setup.py',
-                  script_args=["build_ext", 
-                               '--build-lib=%s' % moduleDir, 
-                               ])
-        os.chdir(oldCwd)
+
+        script_args = [sys.executable, '%s%ssetup.py' % (moduleDir, os.sep)]
+
+        if verbose:
+            script_args.append('--verbose')
+        else:
+            script_args.append('--quiet')
+
+        script_args.extend(['build_ext', '--build-lib=%s' % moduleDir])
+
+        # distutils.core.run_setup looks nicer, but does not let us check the
+        # result easily
+        try:
+            result = subprocess.run(script_args,
+                                    cwd=moduleDir,
+                                    stdout=subprocess.PIPE,
+                                    stderr=subprocess.STDOUT,
+                                    check=True)
+        except subprocess.CalledProcessError as e:
+            print(e.output.decode('utf-8'))
+            raise
+
+        if verbose:
+            print(result.stdout.decode('utf-8'))
 
     def writeIndexFiles(self,name):
         """Write index file for a symbolic array.

python/amici/setup.template.py

@@ -26,17 +26,18 @@ def getAmiciLibs():
 import pkgconfig
 h5pkgcfg = pkgconfig.parse("hdf5")
 
-cxx_flags = ['-std=c++0x']
+cxx_flags = ['-std=c++11']
 #linker_flags = ['${BLAS_LIBRARIES}']
 linker_flags = []
 if 'ENABLE_GCOV_COVERAGE' in os.environ and os.environ['ENABLE_GCOV_COVERAGE'] == 'TRUE':
     cxx_flags.extend(['-g', '-O0',  '--coverage'])
     linker_flags.append('--coverage')
 
-libraries = ['cblas',# TODO generic BLAS
+libraries = [*getAmiciLibs(),
+             'cblas',# TODO generic BLAS
              'hdf5_hl_cpp', 'hdf5_hl', 'hdf5_cpp', 'hdf5'] 
 sources = ['swig/TPL_MODELNAME.i', *getModelSources()]
-libraries.extend(getAmiciLibs())
+
 
 
 # Remove the "-Wstrict-prototypes" compiler option, which isn't valid for