Propagation of master branch changes

Author: llaniewski

.gitignore

@@ -23,4 +23,5 @@ p
 /*.txt
 .idea/*
 *.ipynb_checkpoint*
+test-*/
 tools/.format/*

example/python/karman_vtk.xml

@@ -0,0 +1,62 @@
+<?xml version="1.0"?>
+<CLBConfig version="2.0" output="output/" permissive="true">
+	<Geometry nx="1024" ny="100">
+		<MRT>
+			<Box/>
+		</MRT>
+		<WVelocity name="Inlet">
+			<Inlet/>
+		</WVelocity>
+		<EPressure name="Outlet">
+			<Outlet/>
+		</EPressure>
+		<Inlet nx="1" dx="5">
+			<Box/>
+		</Inlet>
+		<Outlet nx="1" dx="-5">
+			<Box/>
+		</Outlet>
+		<Wall mask="ALL">
+			<Channel/>
+			<Wedge dx="120" nx="20" dy="50" ny="20" direction="LowerRight"/>
+			<Wedge dx="120" nx="20" dy="30" ny="20" direction="UpperRight"/>
+			<Wedge dx="140" nx="20" dy="50" ny="20" direction="LowerLeft"/>
+			<Wedge dx="140" nx="20" dy="30" ny="20" direction="UpperLeft"/>
+		</Wall>
+	</Geometry>
+	<Model>
+		<Param name="VelocityX" value="0.01"/>
+		<Param name="Viscosity" value="0.02"/>
+		<Param name="Smag" value="0.16"/>
+		<Param name="PressDiffInObj" value="1"/>
+		<Param name="EOSScale" value="0.05"/>
+		<Param name="Tension" value="0.01"/>
+		<Param name="Coriolis" value="0.001"/>
+		<Param name="SolidAlfa" value="0.166"/>
+		<Param name="FluidAlfa" value="0.01"/>
+		<Param name="InitTemperature" value="0"/>
+		<Param name="InletTemperature" value="1"/>
+	</Model>
+	<RunPython>
+import vtk;
+from vtk.util import numpy_support
+	</RunPython>
+	<RunPython Iterations="1000">
+img = vtk.vtkImageData()
+tab = Solver.Geometry.X
+img.SetDimensions(tab.shape[0]+1, tab.shape[1]+1, tab.shape[2]+1)
+
+for n,tab in Solver.Quantities:
+	vtk_data = numpy_support.numpy_to_vtk(num_array=tab.reshape(-1,order='F'))
+	vtk_data.SetName(n)
+	if len(tab.shape) == 4:
+		vtk_data.SetNumberOfComponents(tab.shape[0])
+	img.GetCellData().AddArray(vtk_data)
+	
+writer = vtk.vtkXMLImageDataWriter()
+writer.SetFileName("test.vti")
+writer.SetInputData(img)
+writer.Update()
+	</RunPython>
+	<Solve Iterations="10000"/>
+</CLBConfig>

example/python/pythonGeom.xml

@@ -207,7 +207,7 @@ class rQuantities : public rWrapper {
 	SEXP Dollar(std::string name) {
 		bool si = false;
 		std::string quant = name;
-		size_t last_index = name.find_last_not_of(".");
+		size_t last_index = name.find_last_of(".");
 		if (last_index != std::string::npos) {
 			std::string result = name.substr(last_index + 1);
 			if (result == "si") {
@@ -460,6 +460,7 @@ class rSolver : public rWrapper {
 		ret.push_back("Globals");
 		ret.push_back("Actions");
 		ret.push_back("Geometry");
+		ret.push_back("Info");
 		return ret;
 	}
 };
@@ -642,13 +643,21 @@ namespace RunPython {
 	}	
 
 	void initializePy() {
+		if (py_initialised) return;
 		RInside& R = RunR::GetR();
 		has_reticulate = R.parseEval("require(reticulate, quietly=TRUE)");
 		if (!has_reticulate) throw std::string("Tried to call Python, but no reticulate installed");
 		py_initialised = true;
 		R.parseEval(
 			"py_names = function(obj) names(obj)                                   \n"
-			"py_element = function(obj, name) `[[`(obj,name)                       \n"
+			"py_element = function(obj, name) {                                    \n"
+			"  ret = `[[`(obj,name)                                                \n"
+			"  if (is.factor(ret)) {                                               \n"
+			"    as.integer(ret) - 1L                                              \n"
+			"  } else {                                                            \n"
+			"    ret                                                               \n"
+			"  }                                                                   \n"
+			"}                                                                     \n"
 			"py_element_assign = function(obj, name, value) `[[<-`(obj,name,value) \n"
 			"r_to_py.CLB = function(x, convert=FALSE) py$S3(reticulate:::py_capsule(x))\n"
 		);
@@ -660,6 +669,9 @@ namespace RunPython {
 			"    return r.print(self.obj)                                          \n"
 			"  def __dir__(self):                                                  \n"
 			"    return r.py_names(self.obj)                                       \n"
+			"  def __iter__(self):                                                 \n"
+			"    for n in r.py_names(self.obj):                                    \n"
+			"      yield n, r.py_element(self.obj, n)                              \n"
 			"  def __getattr__(self, index):                                       \n"
 			"    if index.startswith('_'):                                         \n"
 			"      return None                                                     \n"

example/python/runexternal.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env Rscript
+
+library(optparse)
+options <- list(
+    make_option(c("-o","--output"), "store", default="", help="Output file", type="character"),
+    make_option(c("-c","--continue"), "store", default="Iteration,Time_si,Walltime", help="regexp for columns to continue by addition (default: Iteration,Time_si,Walltime)", type="character"),
+	make_option(c("-d","--discard"), "store", default="", help="regexp for columns to discard (separate by '|')", type="character")
+)
+
+opt <- parse_args(OptionParser(usage="Usage: csvconcatenate -o file file1 file2 file3", options), positional_arguments=TRUE)
+
+read = function(f) {
+	if (! file.exists(f)) stop(paste("File not found:",f))
+	tab = try(read.csv(f),silent=TRUE)
+	if (inherits(tab, "try-error")) stop(paste(f,"is not a valid CSV file"))
+	tab
+}
+
+args = opt$args
+opt = opt$options
+
+opt$discard = strsplit(opt$discard,",")[[1]]
+if (! is.character(opt$discard)) stop("Discard list is not text")
+
+opt$continue = strsplit(opt$continue,",")[[1]]
+if (! is.character(opt$continue)) stop("Continue list is not text")
+
+grepl_any = function(pattern, x, ...) {
+	sel = rep(FALSE,length(x))
+	for (p in pattern) sel = sel | grepl(p, x, ...)
+	sel
+}
+
+if (length(args) < 1) stop("No csv files provided")
+tab = NULL
+for (fn in args) {
+    tab1 = read(fn)
+    sel = grepl_any(opt$discard, names(tab1))
+    if (any(sel)) tab1=tab1[,!sel]
+    if (is.null(tab)) {
+        tab = tab1
+    } else {
+        if ( ! identical(sort(names(tab)), sort(names(tab1))) ) {
+            cat("names (header) not identical:\n")
+            print(sort(names(tab)))
+            print(sort(names(tab1)))
+            q(status=-2)
+        }
+        sel = grepl_any(opt$continue, names(tab1))
+        if (any(sel)) {
+            for (cn in names(tab1)[sel]) {
+                if (! cn %in% names(tab)) stop(cn,"not in previous csv files")
+                tab1[[cn]] = tab1[[cn]] + max(tab[[cn]])
+            }
+        }
+        tab = rbind(tab,tab1)
+    }
+}
+
+if (opt$output != "") {
+    write.csv(tab, opt$output, row.names=FALSE)
+} else {
+    write.csv(tab, row.names=FALSE)
+}
+
+q(status=0);	
+

example/python/runexternal.xml

@@ -172,31 +172,35 @@ export PYTHONPATH="$PYTHONPATH:$PWD/tools/python"
 function runline {
 	CMD=$1
 	shift
+	case $CMD in
+		run) try "running solver" "$@"; return $?;;
+		fail) try "running solver (should fail)" '!' "$@"; return $? ;;
+		csvconcatenate) try "concatenating csv files" $TCLB/tools/csvconcatenate "$@"; return $? ;;
+	esac
 	R=$1
-	G=$TEST_DIR/$R
+	shift
 	case $CMD in
-	need) 
-		comment_wait "copy $@"
-		for i in "$@"
-		do
-			SRC=$TEST_DIR/$i
-			if test -f "$SRC"
-			then
-				cp $SRC $i
-			else
-				comment_fail "copy $@"
-				echo "         $i not found"
-				return -1;
-			fi
-		done
-		comment_ok "copy $@"
-		;;
-	run) try "running solver" "$@" ;;
-	fail) try "running solver" '!' "$@" ;;
-	csvdiff) try "checking $R (csvdiff)" $TCLB/tools/csvdiff -a "$R" -b "$G" -x "${2:-1e-10}" -d ${3:-$CSV_DISCARD} ;;
-	diff) try "checking $R" diff "$R" "$G" ;;
-	sha1) try "checking $R (sha1)" sha1sum -c "$G.sha1" ;;
-	pvtidiff) try "checking $R (pvtidiff)" $TCLB/CLB/$MODEL/compare "$R" "$G" "${2:-8}" ${3:-} ${4:-} ${5:-} ;; # ${2:-8} is { if $2 == "" then "8" else $2 }
+		exists) try "checking $R (exists)" test -f "$R"; return $? ;;
+		sha1) G="$R.sha1" ;;
+		*) G="$R" ;;
+	esac
+	if test -f "$TEST_DIR/$1"
+	then
+		G="$1"
+		shift
+	fi
+	G="$TEST_DIR/$G"
+	if ! test -f "$G"
+	then
+		comment_fail "Requested file not found: $G"
+		return -1
+	fi
+	case $CMD in
+	need) try "copy needed file" cp "$G" "$R"; return $? ;;
+	csvdiff) try "checking $R (csvdiff)" $TCLB/tools/csvdiff -a "$R" -b "$G" -x "${1:-1e-10}" -d ${2:-$CSV_DISCARD}; return $? ;;
+	diff) try "checking $R" diff "$R" "$G"; return $? ;;
+	sha1) try "checking $R (sha1)" sha1sum -c "$G.sha1"; return $? ;;
+	pvtidiff) try "checking $R (pvtidiff)" $TCLB/CLB/$MODEL/compare "$R" "$G" "${1:-8}" ${2:-} ${3:-} ${4:-}; return $? ;; # ${2:-8} is { if $2 == "" then "8" else $2 }
 	*) echo "unknown: $CMD"; return -1;;
 	esac
 	return 0;
@@ -214,6 +218,7 @@ function testModel {
 		TCLB=".."
 		SOLVER="$TCLB/CLB/$MODEL/main"
 		MODELBIN="$TCLB/CLB/$MODEL"
+		TOOLS="$TCLB/tools"
 		TEST_DIR="../tests/external/$MODEL"
 		CAN_FAIL=false
 		CSV_DISCARD=Walltime