diff --git a/ndsl/dsl/dace/dace_config.py b/ndsl/dsl/dace/dace_config.py index 8f3d2688..78dcb7a5 100644 --- a/ndsl/dsl/dace/dace_config.py +++ b/ndsl/dsl/dace/dace_config.py @@ -204,12 +204,25 @@ def __init__( "openmp_sections", value=0, ) + # Resolve "march/mtune" option + # - turn numeric-centric SSE by default + # - Neoverse-V2 Grace CPU will fail - use alternative mcpu=native. + # Detecting neoverse-v1/2 requires an external package, we swap it + # for a read on GH200 nodes themselves + march_option = "-Xcompiler -march=native" + if ( + cp is not None + and cp.cuda.runtime.getDeviceProperties(0)["name"] + == b"NVIDIA GH200 480GB" + ): + march_option = "-Xcompiler -mcpu=native" + # Removed --fast-math dace.config.Config.set( "compiler", "cuda", "args", - value="-std=c++14 -Xcompiler -fPIC -O3 -Xcompiler -march=native", + value=f"-std=c++14 -Xcompiler -fPIC -O3 {march_option}", ) cuda_sm = 60 diff --git a/ndsl/logging.py b/ndsl/logging.py index 2ec6f979..5e7c845e 100644 --- a/ndsl/logging.py +++ b/ndsl/logging.py @@ -8,7 +8,7 @@ from mpi4py import MPI -LOGLEVEL = os.environ.get("PACE_LOGLEVEL", "INFO").upper() +LOGLEVEL = os.environ.get("PACE_LOGLEVEL", "INFO").lower() # Python log levels are hierarchical, therefore setting INFO # means DEBUG and everything lower will be logged. @@ -23,10 +23,10 @@ def _ndsl_logger() -> logging.Logger: name_log = logging.getLogger(__name__) - name_log.setLevel(LOGLEVEL) + name_log.setLevel(AVAILABLE_LOG_LEVELS[LOGLEVEL]) handler = logging.StreamHandler(sys.stdout) - handler.setLevel(LOGLEVEL) + handler.setLevel(AVAILABLE_LOG_LEVELS[LOGLEVEL]) formatter = logging.Formatter( fmt=( f"%(asctime)s|%(levelname)s|rank {MPI.COMM_WORLD.Get_rank()}|" @@ -41,13 +41,13 @@ def _ndsl_logger() -> logging.Logger: def _ndsl_logger_on_rank_0() -> logging.Logger: name_log = logging.getLogger(f"{__name__}_on_rank_0") - name_log.setLevel(LOGLEVEL) + name_log.setLevel(AVAILABLE_LOG_LEVELS[LOGLEVEL]) rank = MPI.COMM_WORLD.Get_rank() if rank == 0: handler = logging.StreamHandler(sys.stdout) - handler.setLevel(LOGLEVEL) + handler.setLevel(AVAILABLE_LOG_LEVELS[LOGLEVEL]) formatter = logging.Formatter( fmt=( f"%(asctime)s|%(levelname)s|rank {MPI.COMM_WORLD.Get_rank()}|"