wraped up the PR

 * updated doc for BUILD.sh and set RAJA and UMPIRE to OFF default values for the build system

 * removed addToSymDenseMatrixUpperTriangle and documented transAddToSymDenseMatrixUpperTriangle

  * added unit tests for addUpperTriangleToSymDenseMatrixUpperTriangle.

  * updated README with minimal RAJA/UMPIRE information
  * put assert / warnings in LA factory when mem space is not available

Author: cnpetra

BUILD.sh

@@ -1,5 +1,20 @@
 #!/bin/bash
 
+#
+# A script file that builds and runs/tests HiOp on various paricular clusters, such as
+# summit and ascent @ ORNL and newell and marianas @ PNNL
+#
+# Usage: In a shell run
+#
+# ./BUILD.sh
+#
+#
+# Sometimes the cluster name is not detected correctly; in this cases, one can specify
+# the cluster name by prefixing the command with MY_CLUSTER=cluster_name, e.g., 
+#
+# MY_CLUSTER=ascent ./BUILD.sh
+#
+
 cleanup() {
   echo
   echo Exit code $1 caught in build script.

CMakeLists.txt

@@ -22,8 +22,8 @@ option(HIOP_USE_GPU "Build with support for GPUs - Magma and cuda libraries" OFF
 option(HIOP_TEST_WITH_BSUB "Use `jsrun` instead of `mpirun` commands when running tests" OFF)
 option(HIOP_USE_UMPIRE "Build with portable memory management library Umpire" ON)
 option(HIOP_USE_RAJA   "Build with portability abstraction library RAJA" ON)
-option(HIOP_DEEPCHECKS "Extra checks and asserts in the code with a high penalty on performance" ON)
-option(HIOP_WITH_KRON_REDUCTION "Build Kron Reduction code (requires MA86)" OFF)
+option(HIOP_DEEPCHECKS "Extra checks and asserts in the code with a high penalty on performance" OFF)
+option(HIOP_WITH_KRON_REDUCTION "Build Kron Reduction code (requires UMFPACK)" OFF)
 option(HIOP_DEVELOPER_MODE "Build with extended warnings and options" OFF)
 #with testing drivers capable of 'selfchecking' (-selfcheck)
 option(HIOP_WITH_MAKETEST "Enable 'make test'" ON)

README.md

@@ -18,9 +18,11 @@ $> cmake -DCMAKE_INSTALL_PREFIX=/usr/lib/hiop ..'
 ```
 
 
-### HiOp-specific build options
+### Selected HiOp-specific build options
 * Enable/disable MPI: *-DHIOP_USE_MPI=[ON/OFF]* (by default ON)
 * GPU support: *-DHIOP_USE_GPU=ON*. MPI can be either off or on. For more build system options related to GPUs, see "Dependencies" section below.
+* Use RAJA portability layer to allow running linear algebra in either host (CPU) or device (GPU): *-DHIOP_USE_RAJA=ON* and *-DHIOP_USE_UMPIRE=ON*. These build options are off by default. Currently, HiOp only supports unified memory space.
+* Enable/disable "developer mode" build that enforces more restrictive compiler rules and guidelines: *-DHIOP_DEVELOPER_MODE=ON*. This option is by default off.
 * Additional checks and self-diagnostics inside HiOp meant to detect anormalities and help to detect bugs and/or troubleshoot problematic instances: *-DHIOP_DEEPCHECKS=[ON/OFF]* (by default ON). Disabling HIOP_DEEPCHECKS usually provides 30-40% execution speedup in HiOp. For full strength, it is recomended to use HIOP_DEEPCHECKS with debug builds. With non-debug builds, in particular the ones that disable the assert macro, HIOP_DEEPCHECKS does not perform all checks and, thus, may overlook potential issues.
 
 For example:
@@ -41,6 +43,8 @@ HiOp requires LAPACK and BLAS. These dependencies are automatically detected by
 
 HiOp has some support for NVIDIA **GPU-based computations** via CUDA and Magma. To enable the use of GPUs,  use cmake with '-DHIOP_USE_GPU=ON'. The build system will automatically search for CUDA Toolkit. For non-standard CUDA Toolkit installations, use '-DHIOP_CUDA_LIB_DIR=/path' and '-DHIOP_CUDA_INCLUDE_DIR=/path'. For "very" non-standard CUDA Toolkit installations, one can specify the directory of cuBlas libraries as well with '-DHIOP_CUBLAS_LIB_DIR=/path'.
 
+When RAJA-based portability abstraction layer is enabled, HiOp requires RAJA and UMPIRE libraries 
+
 ### Support for GPU computations
 
 When GPU support is on, HiOp requires Magma and CUDA Toolkit. Both are detected automatically in most normal use. The typical cmake command to enable GPU support in HiOp is
@@ -67,6 +71,8 @@ $> cmake -DHIOP_USE_GPU=ON -DCUDA_cublas_LIBRARY=/usr/local/cuda-10.2/targets/x8
 
 A detailed example on how to compile HiOp straight of the box on `summit.olcf.ornl.gov` is available [here](README_summit.md).
 
+RAJA and UMPIRE dependencies are usually detected by HiOp's cmake build system. 
+
 ### Kron reduction
 
 Kron reduction functionality of HiOp is disabled by default. One can enable it by using 
@@ -85,7 +91,7 @@ HiOp supports two input formats: `hiopInterfaceDenseConstraints` and `hiopInterf
 
 *`hiopInterfaceDenseConstraints` interface* supports NLPs with **billions** of variables with and without bounds but only limited number (<100) of general, equality and inequality constraints. The underlying algorithm is a limited-memory quasi-Newton interior-point method and generally scales well computationally (but it may not algorithmically) on thousands of cores. This interface uses MPI for parallelization
 
-*`hiopInterfaceMDS` interface* supports mixed dense-sparse NLPs and achives parallelization using GPUs. Limited speed-up can be obtained on multi-cores CPUs via multithreaded MKL. 
+*`hiopInterfaceMDS` interface* supports mixed dense-sparse NLPs and achives parallelization using GPUs and RAJA portability abstraction layer. 
 
 More information on the HiOp interfaces are [here](src/Interface/README.md).
 
@@ -111,7 +117,8 @@ HiOp has been developed under the financial support of:
 - Lawrence Livermore National Laboratory, through the LDRD program
 
 # Copyright
-Copyright (c) 2017, Lawrence Livermore National Security, LLC. All rights reserved. Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-742473. Written by Cosmin G. Petra, [email protected]. 
+Copyright (c) 2017, Lawrence Livermore National Security, LLC. All rights reserved. Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-742473. 
+HiOp is written by Cosmin G. Petra ([email protected]) of LLNL and has received contributions from Slaven Peles (PNNL), Asher Mancinelli (PNNL), Jake K. Ryan (PNNL), Cameron Rutherford (PNNL), Nai-Yuan Chiang (LLNL), and Michel Schanen (ANL).
 
 HiOp is free software; you can modify it and/or redistribute it under the terms of the BSD 3-clause license. See [COPYRIGHT](/COPYRIGHT) and [LICENSE](/LICENSE) for complete copyright and license information.
  

src/Drivers/nlpMDS_ex4_raja_driver.cpp

@@ -16,8 +16,6 @@
 
 using namespace hiop;
 
-static bool self_check(long long n, double obj_value);
-
 static bool parse_arguments(int argc, char **argv,
 			    bool& self_check,
 			    long long& n_sp,

src/LinAlg/hiopLinAlgFactory.cpp

@@ -95,6 +95,10 @@ hiopVector* LinearAlgebraFactory::createVector(
   {
 #ifdef HIOP_USE_RAJA
     return new hiopVectorRajaPar(glob_n, mem_space_, col_part, comm);
+#else
+    assert(false && "requested memory space not available because Hiop was not"
+           "built with RAJA support");
+    return new hiopVectorPar(glob_n, col_part, comm);
 #endif
   }
 }
@@ -115,6 +119,10 @@ hiopVectorInt* LinearAlgebraFactory::createVectorInt(int size)
   {
 #ifdef HIOP_USE_RAJA
     return new hiopVectorIntRaja(size, mem_space_);
+#else
+    assert(false && "requested memory space not available because Hiop was not"
+           "built with RAJA support");
+    return new hiopVectorIntSeq(size);
 #endif
   }
 }
@@ -140,6 +148,10 @@ hiopMatrixDense* LinearAlgebraFactory::createMatrixDense(
   {
 #ifdef HIOP_USE_RAJA
     return new hiopMatrixRajaDense(m, glob_n, mem_space_, col_part, comm, m_max_alloc);
+#else
+    assert(false && "requested memory space not available because Hiop was not"
+           "built with RAJA support");
+    return new hiopMatrixDenseRowMajor(m, glob_n, col_part, comm, m_max_alloc);
 #endif
   }
 
@@ -159,6 +171,10 @@ hiopMatrixSparse* LinearAlgebraFactory::createMatrixSparse(int rows, int cols, i
   {
 #ifdef HIOP_USE_RAJA
     return new hiopMatrixRajaSparseTriplet(rows, cols, nnz, mem_space_);
+#else
+    assert(false && "requested memory space not available because Hiop was not"
+           "built with RAJA support");
+    return new hiopMatrixSparseTriplet(rows, cols, nnz);
 #endif
   }
 }
@@ -177,6 +193,10 @@ hiopMatrixSparse* LinearAlgebraFactory::createMatrixSymSparse(int size, int nnz)
   {
 #ifdef HIOP_USE_RAJA
     return new hiopMatrixRajaSymSparseTriplet(size, nnz, mem_space_);
+#else
+    assert(false && "requested memory space not available because Hiop was not"
+           "built with RAJA support");
+    return new hiopMatrixSymSparseTriplet(size, nnz);
 #endif
   }
 }
@@ -196,6 +216,10 @@ double* LinearAlgebraFactory::createRawArray(int n)
     auto& resmgr = umpire::ResourceManager::getInstance();
     umpire::Allocator al  = resmgr.getAllocator(mem_space_);
     return static_cast<double*>(al.allocate(n*sizeof(double)));
+#else
+    assert(false && "requested memory space not available because Hiop was not"
+           "built with RAJA support");
+    return new double[n];
 #endif
   }
 }

src/LinAlg/hiopMatrix.hpp

@@ -128,38 +128,32 @@ class hiopMatrix
   /// @brief this += alpha*X
   virtual void addMatrix(double alpha, const hiopMatrix& X) = 0;
 
-  /**
-   * @brief block of W += alpha*this
-   *
-   * For efficiency, only upper triangular matrix is updated since this will be eventually sent to LAPACK
-   *
-   * @pre 'this' fits in the upper triangle of W 
-   * @pre W.n() == W.m()
-   * @pre 'this' and W are local/non-distributed matrices
-   */
-  virtual void addToSymDenseMatrixUpperTriangle(int row_dest_start, int col_dest_start, 
-						double alpha, hiopMatrixDense& W) const = 0;
-
   /**
    * @brief block of W += alpha*transpose(this)
    *
    * For efficiency, only upper triangular matrix is updated since this will be eventually sent to LAPACK
    *
+   * The functionality of this method is needed only for general (non-symmetric) matrices and, for this
+   * reason, only general matrices classes implement/need to implement this method.
+   *
    * @pre transpose of 'this' fits in the upper triangle of W 
    * @pre W.n() == W.m()
    * @pre 'this' and W are local/non-distributed matrices
    */
   virtual void transAddToSymDenseMatrixUpperTriangle(int row_dest_start, int col_dest_start, 
-						     double alpha, hiopMatrixDense& W) const = 0;
+  						     double alpha, hiopMatrixDense& W) const = 0;
 
   /**
    * @brief diagonal block of W += alpha*this with 'diag_start' indicating the diagonal entry of W where
-   * 'this' should start to contribute.
+   * 'this' should start to contribute to.
    * 
    * For efficiency, only upper triangle of W is updated since this will be eventually sent to LAPACK
-   * and only the upper triangle of 'this' is accessed
+   * and only the upper triangle of 'this' is accessed.
    * 
-   * @pre this->n()==this-m()
+   * This functionality of this method is needed only for symmetric matrices and, for this reason,
+   * only symmetric matrices classes implement/need to implement it.
+   *
+   * @pre this->n()==this->m()
    * @pre W.n() == W.m()
    * @pre 'this' and W are local/non-distributed matrices
    */

src/LinAlg/hiopMatrixComplexDense.hpp

@@ -24,7 +24,6 @@ namespace hiop
     - timesMatTran_local
     - addDiagonal (both overloads)
     - addSubDiagonal (all three overloads)
-    - addToSymDenseMatrixUpperTriangle
     - transAddToSymDenseMatrixUpperTriangle
     - addUpperTriangleToSymDenseMatrixUpperTriangle
     - copyRowsFrom
@@ -155,17 +154,6 @@ namespace hiop
     void addSparseSymUpperTriangleToSymDenseMatrixUpperTriangle(const std::complex<double>& alpha,
 								const hiopMatrixComplexSparseTriplet& X);
 
-    /* block of W += alpha*this
-     * For efficiency, only upper triangular matrix is updated since this will be eventually sent to LAPACK
-     * Preconditions: 
-     *  1. 'this' has to fit in the upper triangle of W 
-     *  2. W.n() == W.m()
-     */
-    virtual void addToSymDenseMatrixUpperTriangle(int row_dest_start, int col_dest_start, 
-						  double alpha, hiopMatrixDense& W) const
-    {
-      assert(false && "not supported");
-    }
     /* block of W += alpha*transpose(this)
      * For efficiency, only upper triangular matrix is updated since this will be eventually sent to LAPACK
      * Preconditions: 

src/LinAlg/hiopMatrixDense.hpp

@@ -135,15 +135,6 @@ class hiopMatrixDense : public hiopMatrix
 
   virtual void addMatrix(double alpha, const hiopMatrix& X){assert(false && "not implemented in base class");}
 
-  /**
-   * @brief block of W += alpha*this
-   * For efficiency, only upper triangular matrix is updated since this will be eventually sent to LAPACK
-   *
-   * @pre 'this' has to fit in the upper triangle of W 
-   * @pre W.n() == W.m()
-   */
-  virtual void addToSymDenseMatrixUpperTriangle(int row_dest_start, int col_dest_start, 
-						double alpha, hiopMatrixDense& W) const{assert(false && "not implemented in base class");}
   /**
    * @brief block of W += alpha*transpose(this)
    * For efficiency, only upper triangular matrix is updated since this will be eventually sent to LAPACK
@@ -152,7 +143,10 @@ class hiopMatrixDense : public hiopMatrix
    * @pre W.n() == W.m()
    */
   virtual void transAddToSymDenseMatrixUpperTriangle(int row_dest_start, int col_dest_start, 
-						     double alpha, hiopMatrixDense& W) const{assert(false && "not implemented in base class");}
+						     double alpha, hiopMatrixDense& W) const
+  {
+    assert(false && "not implemented in base class");
+  }
 
   /**
    * @brief diagonal block of W += alpha*this with 'diag_start' indicating the diagonal entry of W where

src/LinAlg/hiopMatrixDenseRowMajor.cpp

@@ -714,26 +714,6 @@ void hiopMatrixDenseRowMajor::addMatrix(double alpha, const hiopMatrix& X_)
   DAXPY(&N, &alpha, X.M_[0], &inc, M_[0], &inc);
 }
 
-/* block of W += alpha*this 
- * starts are in destination */
-void hiopMatrixDenseRowMajor::addToSymDenseMatrixUpperTriangle(int row_start, int col_start, 
-						       double alpha, hiopMatrixDense& W) const
-{
-  assert(row_start>=0 && m()+row_start<=W.m());
-  assert(col_start>=0 && n()+col_start<=W.n());
-  assert(W.n()==W.m());
-
-  double** WM = W.get_M();
-  for(int i=0; i<m_local_; i++) {
-    const int iW = i+row_start;
-    for(int j=0; j<n_local_; j++) {
-      const int jW = j+col_start;
-      assert(iW<=jW && "source entries need to map inside the upper triangular part of destination");
-      WM[iW][jW] += alpha*this->M_[i][j];
-    }
-  }
-}
-
 /* block of W += alpha*this' */
 void hiopMatrixDenseRowMajor::transAddToSymDenseMatrixUpperTriangle(int row_start, int col_start, 
 							    double alpha, hiopMatrixDense& W) const

src/LinAlg/hiopMatrixDenseRowMajor.hpp

@@ -136,16 +136,6 @@ class hiopMatrixDenseRowMajor : public hiopMatrixDense
 
   virtual void addMatrix(double alpha, const hiopMatrix& X);
 
-  /**
-   * @brief block of W += alpha*this
-   * For efficiency, only upper triangular matrix is updated since this will be eventually sent to LAPACK
-   *
-   * @pre 'this' has to fit in the upper triangle of W 
-   * @pre W.n() == W.m()
-   */
-  virtual void addToSymDenseMatrixUpperTriangle(int row_dest_start, int col_dest_start, 
-						double alpha, hiopMatrixDense& W) const;
-
   /**
    * @brief block of W += alpha*transpose(this)
    * For efficiency, only upper triangular matrix is updated since this will be eventually sent to LAPACK
@@ -163,6 +153,9 @@ class hiopMatrixDenseRowMajor : public hiopMatrixDense
    * For efficiency, only upper triangle of W is updated since this will be eventually sent to LAPACK
    * and only the upper triangle of 'this' is accessed
    * 
+   * This functionality of this method is needed only for symmetric matrices and, for this reason,
+   * only symmetric matrices classes implement/need to implement it.
+   *
    * @pre this->n()==this->m()
    * @pre W.n() == W.m()
    */

src/LinAlg/hiopMatrixMDS.hpp

@@ -133,11 +133,12 @@ class hiopMatrixMDS : public hiopMatrix
   }
 
   /* block of W += alpha*this */
-  virtual void addToSymDenseMatrixUpperTriangle(int row_start, int col_start, double alpha, hiopMatrixDense& W) const
-  {
-    mSp->addToSymDenseMatrixUpperTriangle(row_start, col_start, alpha, W);
-    mDe->addToSymDenseMatrixUpperTriangle(row_start, col_start+mSp->n(), alpha, W);
-  }
+  // virtual void addToSymDenseMatrixUpperTriangle(int row_start, int col_start, double alpha, hiopMatrixDense& W) const
+  // {
+  //   mSp->addToSymDenseMatrixUpperTriangle(row_start, col_start, alpha, W);
+  //   mDe->addToSymDenseMatrixUpperTriangle(row_start, col_start+mSp->n(), alpha, W);
+  // } aaa
+
   /* block of W += alpha*this' */
   virtual void transAddToSymDenseMatrixUpperTriangle(int row_start, int col_start, double alpha, hiopMatrixDense& W) const
   {
@@ -158,7 +159,7 @@ class hiopMatrixMDS : public hiopMatrix
   virtual void addUpperTriangleToSymDenseMatrixUpperTriangle(int diag_start, 
 							     double alpha, hiopMatrixDense& W) const
   {
-    assert(false && "not for general matrices; counterpart method from hiopMatrixSymBlockDiagMDS should be used instead");
+    assert(false && "not needed for general/nonsymmetric matrices.");
   }
 
   virtual double max_abs_value()
@@ -238,7 +239,6 @@ class hiopMatrixMDS : public hiopMatrix
  * - timesMatTran
  * - addDiagonal (both overloads)
  * - addSubDiagonal (all three overloads)
- * - addToSymDenseMatrixUpperTriangle
  * - transAddToSymDenseMatrixUpperTriangle
  */
 class hiopMatrixSymBlockDiagMDS : public hiopMatrix
@@ -346,26 +346,13 @@ class hiopMatrixSymBlockDiagMDS : public hiopMatrix
   /**
    *  block of W += alpha*this
    * 
-   * @todo Should this method be available at all? Consider using
-   * addUpperTriangleToSymDenseMatrixUpperTriangle instead.
-   */
-  virtual void addToSymDenseMatrixUpperTriangle(int row_start, int col_start, double alpha, hiopMatrixDense& W) const
-  {
-    assert(mSp->m() == mSp->n());
-    mSp->addToSymDenseMatrixUpperTriangle(row_start,          col_start,          alpha, W);
-    mDe->addToSymDenseMatrixUpperTriangle(row_start+mSp->n(), col_start+mSp->n(), alpha, W);
-    assert(0 && "This should not be called for symmetric matrices.");
-  }
-  /**
-   *  block of W += alpha*this
-   * 
-   * @warning This method is implemented correctly, but should never be called actually.
+   * @warning This method should never be called/is never needed for symmetric matrixes.
    * Use addUpperTriangleToSymDenseMatrixUpperTriangle instead.
    */
-  virtual void transAddToSymDenseMatrixUpperTriangle(int row_start, int col_start, double alpha, hiopMatrixDense& W) const
+  virtual void transAddToSymDenseMatrixUpperTriangle(int row_start, int col_start, 
+						     double alpha, hiopMatrixDense& W) const
   {
-    addToSymDenseMatrixUpperTriangle(row_start, col_start, alpha, W);
-    assert(0 && "This should not be called for symmetric matrices.");
+    assert(0 && "This should not be called for MDS symmetric matrices.");
   }
 
   /* diagonal block of W += alpha*this with 'diag_start' indicating the diagonal entry of W where
@@ -375,7 +362,7 @@ class hiopMatrixSymBlockDiagMDS : public hiopMatrix
    * and only the upper triangle of 'this' is accessed
    * 
    * Preconditions: 
-   *  1. this->n()==this-m()
+   *  1. this->n()==this->m()
    *  2. W.n() == W.m()
    */
   virtual void addUpperTriangleToSymDenseMatrixUpperTriangle(int diag_start,