From a41c2e80ee0d37389c0c5e8c60255fbd8d40149a Mon Sep 17 00:00:00 2001 From: yanqingmen Date: Thu, 17 Dec 2015 04:52:35 -0800 Subject: [PATCH 01/14] add IO classes --- .gitignore | 34 ++++++----- .../src/main/scala/ml/dmlc/mxnet/Base.scala | 1 + .../src/main/scala/ml/dmlc/mxnet/IO.scala | 57 +++++++++++++++++++ .../main/scala/ml/dmlc/mxnet/LibInfo.scala | 8 +++ 4 files changed, 82 insertions(+), 18 deletions(-) create mode 100644 scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala diff --git a/.gitignore b/.gitignore index 7da5b5743768..14ea262ae8c3 100644 --- a/.gitignore +++ b/.gitignore @@ -88,23 +88,23 @@ example/notebooks/.ipynb_checkpoints/* # Scala package # Jetbrain -mxnet-scala/.idea +scala-package/.idea # ctags -mxnet-scala/tags +scala-package/tags -mxnet-scala/*.class -mxnet-scala/*.log +scala-package/*.class +scala-package/*.log # sbt specific -mxnet-scala/.cache -mxnet-scala/.lib/ -mxnet-scala/dist/* -mxnet-scala/target/ -mxnet-scala/lib_managed/ -mxnet-scala/src_managed/ -mxnet-scala/project/boot/ -mxnet-scala/project/plugins/project/ +scala-package/.cache +scala-package/.lib/ +scala-package/dist/* +scala-package/target/ +scala-package/lib_managed/ +scala-package/src_managed/ +scala-package/project/boot/ +scala-package/project/plugins/project/ #scala target folders scala-package/*/target/ @@ -116,9 +116,7 @@ scala-package/*/*/target/ .settings # IDE specific -mxnet-scala/.scala_dependencies -mxnet-scala/.worksheet -mxnet-scala/.idea -mxnet-scala/*.iml - - +.scala_dependencies +.worksheet +.idea +*.iml \ No newline at end of file diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/Base.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/Base.scala index d3be0b2bd453..82bfca87454b 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/Base.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/Base.scala @@ -16,6 +16,7 @@ object Base { type MXFloatRef = RefFloat type NDArrayHandle = RefLong type FunctionHandle = RefLong + type DataIterHandle = RefLong System.loadLibrary("mxnet-scala") val _LIB = new LibInfo diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala new file mode 100644 index 000000000000..a532b7277104 --- /dev/null +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala @@ -0,0 +1,57 @@ +package ml.dmlc.mxnet + +import ml.dmlc.mxnet.Base._ +import ml.dmlc.mxnet.NDArray +import org.slf4j.LoggerFactory + + + +abstract class DataIter (val batchSize: Int = 0) { + /** + * reset the iterator + */ + def reset(): Unit + /** + * Iterate to next batch + * @return whether the move is successful + */ + def iterNext(): Boolean + + /** + * get data of current batch + * @return the data of current batch + */ + def getData(): NDArray + + /** + * Get label of current batch + * @return the label of current batch + */ + def getLabel(): NDArray + + /** + * get the number of padding examples + * in current batch + * @return number of padding examples in current batch + */ + def getPad(): Int + + /** + * the index of current batch + * @return + */ + def getIndex(): Seq[Int] +} + +class MXDataIter(var handle: DataIterHandle) extends DataIter { + private val logger = LoggerFactory.getLogger(classOf[MXDataIter]) + + def reset(): Unit = { + checkCall(_LIB.mxDataIterBeforeFirst(handle)) + } + + def iterNext(): Boolean = { + checkCall(_LIB.mxDataIterNext(handle)) + return true + } +} \ No newline at end of file diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala index fb937c82ee78..f05a5d27028b 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala @@ -47,4 +47,12 @@ class LibInfo { start: MXUint, end: MXUint, sliceHandle: NDArrayHandle): Int + @native def mxDataIterBeforeFirst(handle: DataIterHandle): Int + @native def mxDataIterNext(handle: DataIterHandle): Int + @native def mxDataIterGetLabel(handle: DataIterHandle, + out: NDArrayHandle): Int + @native def mxDataIterGetData(handle: DataIterHandle, + out: NDArrayHandle): Int + @native def mxDataIterGetPadNum(handle: DataIterHandle, + out: MXUintRef): Int } From 3b5e7a38505e7ff4a95d766a86eadea29e0d354d Mon Sep 17 00:00:00 2001 From: yanqingmen Date: Mon, 21 Dec 2015 23:21:31 +0800 Subject: [PATCH 02/14] add MXDataIter --- .../src/main/scala/ml/dmlc/mxnet/Base.scala | 1 + .../src/main/scala/ml/dmlc/mxnet/IO.scala | 61 +++++++++++++++++-- .../main/scala/ml/dmlc/mxnet/LibInfo.scala | 18 +++++- 3 files changed, 74 insertions(+), 6 deletions(-) diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/Base.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/Base.scala index 4febbfb40b16..35a1646a2680 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/Base.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/Base.scala @@ -16,6 +16,7 @@ object Base { type NDArrayHandle = RefLong type FunctionHandle = RefLong type DataIterHandle = RefLong + type DataIterCreator = RefLong type KVStoreHandle = RefLong diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala index a532b7277104..9c4242d6dd59 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala @@ -4,6 +4,7 @@ import ml.dmlc.mxnet.Base._ import ml.dmlc.mxnet.NDArray import org.slf4j.LoggerFactory +import scala.collection.mutable.ListBuffer abstract class DataIter (val batchSize: Int = 0) { @@ -40,18 +41,68 @@ abstract class DataIter (val batchSize: Int = 0) { * the index of current batch * @return */ - def getIndex(): Seq[Int] + def getIndex(): List[Long] } class MXDataIter(var handle: DataIterHandle) extends DataIter { private val logger = LoggerFactory.getLogger(classOf[MXDataIter]) - def reset(): Unit = { + /** + * reset the iterator + */ + override def reset(): Unit = { checkCall(_LIB.mxDataIterBeforeFirst(handle)) } - def iterNext(): Boolean = { - checkCall(_LIB.mxDataIterNext(handle)) - return true + /** + * Iterate to next batch + * @return whether the move is successful + */ + override def iterNext(): Boolean = { + val next = new RefInt + checkCall(_LIB.mxDataIterNext(handle, next)) + return next.value > 0 + } + + /** + * get data of current batch + * @return the data of current batch + */ + override def getData(): NDArray = { + val out = new NDArrayHandle + checkCall(_LIB.mxDataIterGetData(handle, out)) + return new NDArray(out) + } + + /** + * Get label of current batch + * @return the label of current batch + */ + override def getLabel(): NDArray = { + val out = new NDArrayHandle + checkCall(_LIB.mxDataIterGetLabel(handle, out)) + return new NDArray(out) + } + + /** + * the index of current batch + * @return + */ + override def getIndex(): List[Long] = { + val outIndex = new ListBuffer[Long] + val outSize = new RefLong + checkCall(_LIB.mxDataIterGetIndex(handle, outIndex, outSize)) + return outIndex.toList + } + + /** + * get the number of padding examples + * in current batch + * @return number of padding examples in current batch + */ + override def getPad(): MXUint = { + val out = new MXUintRef + checkCall(_LIB.mxDataIterGetPadNum(handle, out)) + return out.value } } \ No newline at end of file diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala index d5b96ab791be..ba9590ed6dea 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala @@ -78,12 +78,28 @@ class LibInfo { @native def mxKVStoreBarrier(handle: KVStoreHandle): Int @native def mxKVStoreGetGroupSize(handle: KVStoreHandle, size: RefInt): Int @native def mxKVStoreGetRank(handle: KVStoreHandle, size: RefInt): Int + //DataIter Funcs + @native def mxListDataIters(handles: Array[DataIterCreator]): Int + @native def mxDateIterCreateIter(handle: DataIterCreator, + keys: Array[String], + vals: Array[String], + out: DataIterHandle): Int + @native def mxDataIterGetIterInfo(creator: DataIterCreator, + name: RefString, + description: RefString, + argNames: ListBuffer[String], + argTypeInfos: ListBuffer[String], + argDescriptions: ListBuffer[String]): Int + @native def mxDataIterFree(handle: DataIterHandle): Int @native def mxDataIterBeforeFirst(handle: DataIterHandle): Int - @native def mxDataIterNext(handle: DataIterHandle): Int + @native def mxDataIterNext(handle: DataIterHandle, out: RefInt): Int @native def mxDataIterGetLabel(handle: DataIterHandle, out: NDArrayHandle): Int @native def mxDataIterGetData(handle: DataIterHandle, out: NDArrayHandle): Int + @native def mxDataIterGetIndex(handle: DataIterHandle, + outIndex: ListBuffer[Long], + outSize: RefLong): Int @native def mxDataIterGetPadNum(handle: DataIterHandle, out: MXUintRef): Int } From f9b68fdb880c8735425c61889a4b06d34643ba5f Mon Sep 17 00:00:00 2001 From: yanqingmen Date: Tue, 22 Dec 2015 22:55:52 +0800 Subject: [PATCH 03/14] add IO funcs --- .../src/main/scala/ml/dmlc/mxnet/IO.scala | 35 +++++++++++++++++-- .../main/scala/ml/dmlc/mxnet/LibInfo.scala | 2 +- .../test/scala/ml/dmlc/mxnet/IOSuite.scala | 11 ++++++ .../main/native/ml_dmlc_mxnet_native_c_api.cc | 24 +++++++++++++ 4 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala index 9c4242d6dd59..014c6e7d4731 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala @@ -1,11 +1,40 @@ package ml.dmlc.mxnet import ml.dmlc.mxnet.Base._ -import ml.dmlc.mxnet.NDArray import org.slf4j.LoggerFactory import scala.collection.mutable.ListBuffer +object IO { + type IterCreateFunc = (Array[String], Array[String])=>DataIter + private val iterCreateFuncs: Map[String, IterCreateFunc] = _initIOModule() + + def _initIOModule(): Map[String, IterCreateFunc] = { + val IterCreators = new ListBuffer[DataIterCreator] + checkCall(_LIB.mxListDataIters(IterCreators)) + IterCreators.map(_makeIOIterator).toMap + } + + def _makeIOIterator(handle: DataIterCreator): (String, IterCreateFunc) = { + val name = new RefString + val desc = new RefString + val argNames = new ListBuffer[String] + val argTypes = new ListBuffer[String] + val argDescs = new ListBuffer[String] + checkCall(_LIB.mxDataIterGetIterInfo(handle, name, desc, argNames, argTypes, argDescs)) + val paramStr = Base.ctypes2docstring(argNames, argTypes, argDescs) + val docStr = s"${name.value}\n${desc.value}\n\n$paramStr\n" + return (name.value, creator(handle)) + } + + def creator(handle:DataIterCreator)( + keys: Array[String], + values: Array[String]): DataIter = { + val out = new DataIterHandle + checkCall(_LIB.mxDateIterCreateIter(handle, keys, values, out)) + return new MXDataIter(out) + } +} abstract class DataIter (val batchSize: Int = 0) { /** @@ -105,4 +134,6 @@ class MXDataIter(var handle: DataIterHandle) extends DataIter { checkCall(_LIB.mxDataIterGetPadNum(handle, out)) return out.value } -} \ No newline at end of file +} + + diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala index ba9590ed6dea..5f2aee984757 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala @@ -79,7 +79,7 @@ class LibInfo { @native def mxKVStoreGetGroupSize(handle: KVStoreHandle, size: RefInt): Int @native def mxKVStoreGetRank(handle: KVStoreHandle, size: RefInt): Int //DataIter Funcs - @native def mxListDataIters(handles: Array[DataIterCreator]): Int + @native def mxListDataIters(handles: ListBuffer[DataIterCreator]): Int @native def mxDateIterCreateIter(handle: DataIterCreator, keys: Array[String], vals: Array[String], diff --git a/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala b/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala new file mode 100644 index 000000000000..35afb27ec315 --- /dev/null +++ b/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala @@ -0,0 +1,11 @@ +package ml.dmlc.mxnet + +import org.scalatest.{BeforeAndAfterAll, FunSuite} + + +class IOSuite extends FunSuite with BeforeAndAfterAll { + test("create iter funcs") { + val iterCreateFuncs: Map[String, IO.IterCreateFunc] = IO._initIOModule() + println(iterCreateFuncs) + } +} diff --git a/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc b/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc index 4e6b1461c945..0e1a704ef45c 100644 --- a/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc +++ b/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc @@ -389,3 +389,27 @@ JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxNDArrayFree(JNIEnv * env, jo return 0; } +//IO funcs +JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxListDataIters + (JNIEnv * env, jobject obj, jobjectArray creators) { + // Base.FunctionHandle.constructor + jclass chClass = env->FindClass("ml/dmlc/mxnet/Base$RefLong"); + jmethodID chConstructor = env->GetMethodID(chClass,"","(J)V"); + + // scala.collection.mutable.ListBuffer append method + jclass listClass = env->FindClass("scala/collection/mutable/ListBuffer"); + jmethodID listAppend = env->GetMethodID(listClass, + "$plus$eq", "(Ljava/lang/Object;)Lscala/collection/mutable/ListBuffer;"); + + // Get function list + DataIterCreator *outArray; + mx_uint outSize; + int ret = MXListDataIters(&outSize, &outArray); + for (int i = 0; i < outSize; ++i) { + DataIterCreator chAddr = outArray[i]; + jobject chObj = env->NewObject(chClass, chConstructor, (long)chAddr); + env->CallObjectMethod(creators, listAppend, chObj); + } + return ret; +} + From 949a611ce7a6dd14c0e16d9305d997a1e2552a2d Mon Sep 17 00:00:00 2001 From: yanqingmen Date: Wed, 23 Dec 2015 22:28:33 +0800 Subject: [PATCH 04/14] add IO jni code --- .../src/main/scala/ml/dmlc/mxnet/IO.scala | 11 +- .../test/scala/ml/dmlc/mxnet/IOSuite.scala | 2 +- .../native/src/main/native/jni_helper_func.h | 5 + .../main/native/ml_dmlc_mxnet_native_c_api.cc | 136 +++++++++++++++++- 4 files changed, 148 insertions(+), 6 deletions(-) diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala index 014c6e7d4731..cb175c39801a 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala @@ -6,7 +6,8 @@ import org.slf4j.LoggerFactory import scala.collection.mutable.ListBuffer object IO { - type IterCreateFunc = (Array[String], Array[String])=>DataIter + private val logger = LoggerFactory.getLogger(classOf[DataIter]) + type IterCreateFunc = (Map[String, String])=>DataIter private val iterCreateFuncs: Map[String, IterCreateFunc] = _initIOModule() def _initIOModule(): Map[String, IterCreateFunc] = { @@ -24,14 +25,16 @@ object IO { checkCall(_LIB.mxDataIterGetIterInfo(handle, name, desc, argNames, argTypes, argDescs)) val paramStr = Base.ctypes2docstring(argNames, argTypes, argDescs) val docStr = s"${name.value}\n${desc.value}\n\n$paramStr\n" + logger.debug(docStr) return (name.value, creator(handle)) } def creator(handle:DataIterCreator)( - keys: Array[String], - values: Array[String]): DataIter = { + params: Map[String, String]): DataIter = { val out = new DataIterHandle - checkCall(_LIB.mxDateIterCreateIter(handle, keys, values, out)) + val keys = params.keys.toArray + val vals = params.values.toArray + checkCall(_LIB.mxDateIterCreateIter(handle, keys, vals, out)) return new MXDataIter(out) } } diff --git a/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala b/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala index 35afb27ec315..34d5ff063a10 100644 --- a/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala +++ b/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala @@ -6,6 +6,6 @@ import org.scalatest.{BeforeAndAfterAll, FunSuite} class IOSuite extends FunSuite with BeforeAndAfterAll { test("create iter funcs") { val iterCreateFuncs: Map[String, IO.IterCreateFunc] = IO._initIOModule() - println(iterCreateFuncs) + println(iterCreateFuncs.keys.toList) } } diff --git a/scala-package/native/src/main/native/jni_helper_func.h b/scala-package/native/src/main/native/jni_helper_func.h index 0d651adf9a94..8354f417ed35 100644 --- a/scala-package/native/src/main/native/jni_helper_func.h +++ b/scala-package/native/src/main/native/jni_helper_func.h @@ -21,4 +21,9 @@ void setIntField(JNIEnv *env, jobject obj, jint value) { env->SetIntField(obj, refFid, value); } +void setLongField(JNIEnv *env, jobject obj, jlong value) { + jclass refClass = env->FindClass("ml/dmlc/mxnet/Base$RefLong"); + jfieldID refFid = env->GetFieldID(refClass, "value", "J"); + env->SetLongField(obj, refFid, value); +} #endif diff --git a/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc b/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc index 0e1a704ef45c..20b76f1238be 100644 --- a/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc +++ b/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc @@ -391,7 +391,7 @@ JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxNDArrayFree(JNIEnv * env, jo //IO funcs JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxListDataIters - (JNIEnv * env, jobject obj, jobjectArray creators) { + (JNIEnv * env, jobject obj, jobject creators) { // Base.FunctionHandle.constructor jclass chClass = env->FindClass("ml/dmlc/mxnet/Base$RefLong"); jmethodID chConstructor = env->GetMethodID(chClass,"","(J)V"); @@ -413,3 +413,137 @@ JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxListDataIters return ret; } +JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxDateIterCreateIter + (JNIEnv * env, jobject obj, jobject creator, + jobjectArray jkeys, jobjectArray jvals, jobject dataIterHandle) { + //keys and values + int paramSize = env->GetArrayLength(jkeys); + char** keys = new char*[paramSize]; + char** vals = new char*[paramSize]; + jstring jkey, jval; + for(int i=0; iGetObjectArrayElement(jkeys, i); + keys[i] = (char*)env->GetStringUTFChars(jkey, 0); + jval = (jstring) env->GetObjectArrayElement(jvals, i); + vals[i] = (char*)env->GetStringUTFChars(jval, 0); + } + + //create iter + jlong creatorPtr = getLongField(env, creator); + DataIterHandle out; + int ret = MXDataIterCreateIter((DataIterCreator)creator, + (mx_uint) paramSize, + (const char**) keys, + (const char**) vals, + &out); + jclass hClass = env->GetObjectClass(dataIterHandle); + jfieldID ptr = env->GetFieldID(hClass, "value", "J"); + env->SetLongField(dataIterHandle, ptr, (long)out); + + //release const char* + for(int i=0; iGetObjectArrayElement(jkeys, i); + env->ReleaseStringUTFChars(jkey,(const char*)keys[i]); + jval = (jstring) env->GetObjectArrayElement(jvals, i); + env->ReleaseStringUTFChars(jval,(const char*)vals[i]); + } + return ret; +} + +JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxDataIterGetIterInfo + (JNIEnv * env, jobject obj, jobject creator, jobject jname, + jobject jdesc, jobject jargNames, jobject jargTypeInfos, jobject jargDescs) { + jlong creatorPtr = getLongField(env, creator); + const char* name; + const char* description; + mx_uint numArgs; + const char** argNames; + const char** argTypeInfos; + const char** argDescs; + int ret = MXDataIterGetIterInfo((DataIterCreator)creatorPtr, + &name, + &description, + &numArgs, + &argNames, + &argTypeInfos, + &argDescs); + + jclass refStringClass = env->FindClass("ml/dmlc/mxnet/Base$RefString"); + jfieldID valueStr = env->GetFieldID(refStringClass, "value", "Ljava/lang/String;"); + //set params + env->SetObjectField(jname, valueStr, env->NewStringUTF(name)); + env->SetObjectField(jdesc, valueStr, env->NewStringUTF(description)); + jclass listClass = env->FindClass("scala/collection/mutable/ListBuffer"); + jmethodID listAppend = env->GetMethodID(listClass, + "$plus$eq", "(Ljava/lang/Object;)Lscala/collection/mutable/ListBuffer;"); + for(int i=0; iCallObjectMethod(jargNames, listAppend, env->NewStringUTF(argNames[i])); + env->CallObjectMethod(jargTypeInfos, listAppend, env->NewStringUTF(argTypeInfos[i])); + env->CallObjectMethod(jargDescs, listAppend, env->NewStringUTF(argDescs[i])); + } + return ret; +} + +JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxDataIterFree + (JNIEnv *env, jobject obj, jobject handle) { + jlong handlePtr = getLongField(env, handle); + int ret = MXDataIterFree((DataIterHandle) handlePtr); + return ret; +} + +JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxDataIterBeforeFirst + (JNIEnv *env, jobject obj, jobject handle) { + jlong handlePtr = getLongField(env, handle); + int ret = MXDataIterBeforeFirst((DataIterHandle) handlePtr); + return ret; +} + +JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxDataIterNext + (JNIEnv *env, jobject obj, jobject handle, jobject out) { + jlong handlePtr = getLongField(env, handle); + int cout; + int ret = MXDataIterNext((DataIterHandle)handlePtr, &cout); + setIntField(env, out, cout); + return ret; +} + +JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxDataIterGetLabel + (JNIEnv *env, jobject obj, jobject handle, jobject ndArrayHandle) { + jlong handlePtr = getLongField(env, handle); + NDArrayHandle out; + int ret = MXDataIterGetLabel((DataIterHandle)handlePtr, &out); + jclass refLongClass = env->FindClass("ml/dmlc/mxnet/Base$RefLong"); + jfieldID refLongFid = env->GetFieldID(refLongClass, "value", "J"); + env->SetLongField(ndArrayHandle, refLongFid, (jlong)out); + return ret; +} + +JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxDataIterGetData + (JNIEnv *env, jobject obj, jobject handle, jobject ndArrayHandle) { + jlong handlePtr = getLongField(env, handle); + NDArrayHandle out; + int ret = MXDataIterGetData((DataIterHandle)handlePtr, &out); + jclass refLongClass = env->FindClass("ml/dmlc/mxnet/Base$RefLong"); + jfieldID refLongFid = env->GetFieldID(refLongClass, "value", "J"); + env->SetLongField(ndArrayHandle, refLongFid, (jlong)out); + return ret; +} + +JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxDataIterGetIndex + (JNIEnv *env, jobject obj, jobject handle, jobject outIndex, jobject outSize) { + jlong handlePtr = getLongField(env, handle); + uint64_t* coutIndex; + uint64_t coutSize; + int ret = MXDataIterGetIndex((DataIterHandle)handlePtr, &coutIndex, &coutSize); + //to do + return ret; +} + +JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxDataIterGetPadNum + (JNIEnv *env, jobject obj, jobject handle, jobject pad) { + jlong handlePtr = getLongField(env, handle); + int cpad; + int ret = MXDataIterGetPadNum((DataIterHandle)handlePtr, &cpad); + setIntField(env, pad, cpad); + return ret; +} \ No newline at end of file From 1bbde69eec968c4b258772ee2d4a711c5d148c00 Mon Sep 17 00:00:00 2001 From: yanqingmen Date: Sat, 26 Dec 2015 23:27:26 +0800 Subject: [PATCH 05/14] add IO MINISITer test --- .../src/main/scala/ml/dmlc/mxnet/IO.scala | 4 +-- .../test/scala/ml/dmlc/mxnet/IOSuite.scala | 32 +++++++++++++++++-- scala-package/native/linux-x86_64/pom.xml | 5 +-- .../main/native/ml_dmlc_mxnet_native_c_api.cc | 23 +++++++++++-- 4 files changed, 54 insertions(+), 10 deletions(-) diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala index cb175c39801a..829a3c1f4fdf 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala @@ -8,7 +8,7 @@ import scala.collection.mutable.ListBuffer object IO { private val logger = LoggerFactory.getLogger(classOf[DataIter]) type IterCreateFunc = (Map[String, String])=>DataIter - private val iterCreateFuncs: Map[String, IterCreateFunc] = _initIOModule() + val iterCreateFuncs: Map[String, IterCreateFunc] = _initIOModule() def _initIOModule(): Map[String, IterCreateFunc] = { val IterCreators = new ListBuffer[DataIterCreator] @@ -25,7 +25,7 @@ object IO { checkCall(_LIB.mxDataIterGetIterInfo(handle, name, desc, argNames, argTypes, argDescs)) val paramStr = Base.ctypes2docstring(argNames, argTypes, argDescs) val docStr = s"${name.value}\n${desc.value}\n\n$paramStr\n" - logger.debug(docStr) +// logger.debug(docStr) return (name.value, creator(handle)) } diff --git a/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala b/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala index 34d5ff063a10..ff6c468ec0ed 100644 --- a/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala +++ b/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala @@ -4,8 +4,34 @@ import org.scalatest.{BeforeAndAfterAll, FunSuite} class IOSuite extends FunSuite with BeforeAndAfterAll { - test("create iter funcs") { - val iterCreateFuncs: Map[String, IO.IterCreateFunc] = IO._initIOModule() - println(iterCreateFuncs.keys.toList) + test("test MNISTIter") { + val params = Map( + "image" -> "/home/hzx/workspace/git/mxnet-scala/mxnet/tests/python/common/data/train-images-idx3-ubyte", + "label" -> "/home/hzx/workspace/git/mxnet-scala/mxnet/tests/python/common/data/train-labels-idx1-ubyte", + "data_shape" -> "(784,)", + "batch_size" -> "100", + "shuffle" -> "1", + "flat" -> "1", + "silent" -> "0", + "seed" -> "10" + ) +// println("create MNISTIter") + val mnist_iter = IO.iterCreateFuncs("MNISTIter")(params) + mnist_iter.reset() + mnist_iter.iterNext() + while(mnist_iter.iterNext()) { + val data = mnist_iter.getData() + val label = mnist_iter.getLabel() + val index = mnist_iter.getIndex() + val pad = mnist_iter.getPad() +// println("data: " + data.toArray.mkString(",")) +// println("label: " + label.toArray.mkString(",")) +// println("index: " + index) +// println("pad: " + pad) + } + } + + test("test ImageRecordIter") { + } } diff --git a/scala-package/native/linux-x86_64/pom.xml b/scala-package/native/linux-x86_64/pom.xml index ef1fd3c0f67b..0d42d47fc173 100644 --- a/scala-package/native/linux-x86_64/pom.xml +++ b/scala-package/native/linux-x86_64/pom.xml @@ -116,8 +116,9 @@ -fopenmp -Wl,--whole-archive - ../../../lib/libmxnet.a - -Wl,-no-whole-archive + ../../../lib/libmxnet.a ../../../dmlc-core/libdmlc.a + + -Wl,--no-whole-archive diff --git a/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc b/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc index 20b76f1238be..2f1c95e0a6d0 100644 --- a/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc +++ b/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc @@ -428,10 +428,16 @@ JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxDateIterCreateIter vals[i] = (char*)env->GetStringUTFChars(jval, 0); } +// printf("paramSize: %d\n", paramSize); +// for(int i=0; iSetLongField(ndArrayHandle, refLongFid, (jlong)out); return ret; } - +l JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxDataIterGetData (JNIEnv *env, jobject obj, jobject handle, jobject ndArrayHandle) { jlong handlePtr = getLongField(env, handle); @@ -535,7 +541,18 @@ JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxDataIterGetIndex uint64_t* coutIndex; uint64_t coutSize; int ret = MXDataIterGetIndex((DataIterHandle)handlePtr, &coutIndex, &coutSize); - //to do + //set field + setLongField(env, outSize, (long)coutSize); + // scala.collection.mutable.ListBuffer append method + jclass listClass = env->FindClass("scala/collection/mutable/ListBuffer"); + jmethodID listAppend = env->GetMethodID(listClass, + "$plus$eq", "(Ljava/lang/Object;)Lscala/collection/mutable/ListBuffer;"); + + printf("outSize: %ld\n", coutSize); + for(int i=0; iCallObjectMethod(outIndex, listAppend, (jlong)coutIndex[i]); + } return ret; } From ee03dae4069704450ce470872286e251250a8861 Mon Sep 17 00:00:00 2001 From: yanqingmen Date: Sun, 27 Dec 2015 11:51:54 +0800 Subject: [PATCH 06/14] add IO ImageRecordIter test --- .../src/main/scala/ml/dmlc/mxnet/IO.scala | 61 ++++++++++++++++++- .../test/scala/ml/dmlc/mxnet/IOSuite.scala | 29 ++++++--- scala-package/native/linux-x86_64/pom.xml | 2 +- .../main/native/ml_dmlc_mxnet_native_c_api.cc | 6 +- 4 files changed, 84 insertions(+), 14 deletions(-) diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala index 829a3c1f4fdf..d293f2215000 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala @@ -7,7 +7,7 @@ import scala.collection.mutable.ListBuffer object IO { private val logger = LoggerFactory.getLogger(classOf[DataIter]) - type IterCreateFunc = (Map[String, String])=>DataIter + type IterCreateFunc = (Map[String, String]) => DataIter val iterCreateFuncs: Map[String, IterCreateFunc] = _initIOModule() def _initIOModule(): Map[String, IterCreateFunc] = { @@ -29,7 +29,7 @@ object IO { return (name.value, creator(handle)) } - def creator(handle:DataIterCreator)( + def creator(handle: DataIterCreator)( params: Map[String, String]): DataIter = { val out = new DataIterHandle val keys = params.keys.toArray @@ -37,6 +37,11 @@ object IO { checkCall(_LIB.mxDateIterCreateIter(handle, keys, vals, out)) return new MXDataIter(out) } + + def _init_data(data: List[NDArray], allowEmpty: Boolean, + defaultName: String): List[Tuple2[]] = { + + } } abstract class DataIter (val batchSize: Int = 0) { @@ -76,6 +81,10 @@ abstract class DataIter (val batchSize: Int = 0) { def getIndex(): List[Long] } +/** + * DataIter built in MXNet. + * @param handle the handle to the underlying C++ Data Iterator + */ class MXDataIter(var handle: DataIterHandle) extends DataIter { private val logger = LoggerFactory.getLogger(classOf[MXDataIter]) @@ -139,4 +148,52 @@ class MXDataIter(var handle: DataIterHandle) extends DataIter { } } +/** + * NDArrayIter object in mxnet. Taking NDArray or numpy array to get dataiter. + * @param data a list of NDArray + * @param label a list of NDArray + * @param batch_size Batch Size + * @param shuffle Whether to shuffle the data + * @param last_batch_handle 'pad', 'discard' or 'roll_over', How to handle the last batch + */ +class NDArrayIter(var data: List[NDArray], var label: List[NDArray], + var batch_size: Int, var shuffle: Boolean, + var last_batch_handle: String) extends DataIter { + /** + * reset the iterator + */ + override def reset(): Unit = ??? + + /** + * get data of current batch + * @return the data of current batch + */ + override def getData(): NDArray = ??? + + /** + * Get label of current batch + * @return the label of current batch + */ + override def getLabel(): NDArray = ??? + + /** + * the index of current batch + * @return + */ + override def getIndex(): List[Long] = ??? + + /** + * Iterate to next batch + * @return whether the move is successful + */ + override def iterNext(): Boolean = ??? + + /** + * get the number of padding examples + * in current batch + * @return number of padding examples in current batch + */ + override def getPad(): MXUint = ??? +} + diff --git a/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala b/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala index ff6c468ec0ed..553f99420f04 100644 --- a/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala +++ b/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala @@ -6,8 +6,8 @@ import org.scalatest.{BeforeAndAfterAll, FunSuite} class IOSuite extends FunSuite with BeforeAndAfterAll { test("test MNISTIter") { val params = Map( - "image" -> "/home/hzx/workspace/git/mxnet-scala/mxnet/tests/python/common/data/train-images-idx3-ubyte", - "label" -> "/home/hzx/workspace/git/mxnet-scala/mxnet/tests/python/common/data/train-labels-idx1-ubyte", + "image" -> "/home/hzx/workspace/git/mxnet-scala/mxnet/tests/python/unittest/data/train-images-idx3-ubyte", + "label" -> "/home/hzx/workspace/git/mxnet-scala/mxnet/tests/python/unittest/data/train-labels-idx1-ubyte", "data_shape" -> "(784,)", "batch_size" -> "100", "shuffle" -> "1", @@ -18,20 +18,33 @@ class IOSuite extends FunSuite with BeforeAndAfterAll { // println("create MNISTIter") val mnist_iter = IO.iterCreateFuncs("MNISTIter")(params) mnist_iter.reset() - mnist_iter.iterNext() while(mnist_iter.iterNext()) { val data = mnist_iter.getData() val label = mnist_iter.getLabel() val index = mnist_iter.getIndex() val pad = mnist_iter.getPad() -// println("data: " + data.toArray.mkString(",")) -// println("label: " + label.toArray.mkString(",")) -// println("index: " + index) -// println("pad: " + pad) } } test("test ImageRecordIter") { - + val params = Map( + "path_imgrec" -> "/home/hzx/workspace/git/mxnet-scala/mxnet/tests/python/unittest/data/cifar/train.rec", + "mean_img" -> "/home/hzx/workspace/git/mxnet-scala/mxnet/tests/python/unittest/data/cifar/cifar10_mean.bin", + "rand_crop" -> "false", + "and_mirror" -> "false", + "shuffle" -> "false", + "data_shape" -> "(3,28,28)", + "batch_size" -> "100", + "preprocess_threads" -> "4", + "prefetch_buffer" -> "1" + ) + val img_iter = IO.iterCreateFuncs("ImageRecordIter")(params) + img_iter.reset() + while(img_iter.iterNext()) { + val data = img_iter.getData() + val label = img_iter.getLabel() + val index = img_iter.getIndex() + val pad = img_iter.getPad() + } } } diff --git a/scala-package/native/linux-x86_64/pom.xml b/scala-package/native/linux-x86_64/pom.xml index 0d42d47fc173..d8de358ca7d4 100644 --- a/scala-package/native/linux-x86_64/pom.xml +++ b/scala-package/native/linux-x86_64/pom.xml @@ -113,7 +113,7 @@ - -fopenmp + -fopenmp -lopencv_calib3d -Wl,--whole-archive ../../../lib/libmxnet.a ../../../dmlc-core/libdmlc.a diff --git a/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc b/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc index 2f1c95e0a6d0..d839a49d4648 100644 --- a/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc +++ b/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc @@ -523,7 +523,7 @@ JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxDataIterGetLabel env->SetLongField(ndArrayHandle, refLongFid, (jlong)out); return ret; } -l + JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxDataIterGetData (JNIEnv *env, jobject obj, jobject handle, jobject ndArrayHandle) { jlong handlePtr = getLongField(env, handle); @@ -548,9 +548,9 @@ JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxDataIterGetIndex jmethodID listAppend = env->GetMethodID(listClass, "$plus$eq", "(Ljava/lang/Object;)Lscala/collection/mutable/ListBuffer;"); - printf("outSize: %ld\n", coutSize); +// printf("outSize: %ld\n", coutSize); for(int i=0; iCallObjectMethod(outIndex, listAppend, (jlong)coutIndex[i]); } return ret; From c89e4ab95609a98c2c4493fa528ce6864a962e95 Mon Sep 17 00:00:00 2001 From: yanqingmen Date: Sun, 27 Dec 2015 23:19:04 +0800 Subject: [PATCH 07/14] add util scriptsfor IO test --- scala-package/core/get_data.sh | 9 +++ .../src/main/scala/ml/dmlc/mxnet/IO.scala | 26 +++++-- .../test/scala/ml/dmlc/mxnet/IOSuite.scala | 69 ++++++++++++------- scala-package/native/linux-x86_64/pom.xml | 14 ++-- 4 files changed, 81 insertions(+), 37 deletions(-) create mode 100755 scala-package/core/get_data.sh diff --git a/scala-package/core/get_data.sh b/scala-package/core/get_data.sh new file mode 100755 index 000000000000..fa10b31ba5ac --- /dev/null +++ b/scala-package/core/get_data.sh @@ -0,0 +1,9 @@ +if [ ! -d "./data" ]; then + mkdir -p ./data +fi +if [ ! -d "./data/mnist.zip" ]; then + wget http://webdocs.cs.ualberta.ca/~bx3/data/mnist.zip -P data/ +fi +cd data +rm *-ubyte +unzip -u mnist.zip \ No newline at end of file diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala index d293f2215000..9021464f2247 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala @@ -25,7 +25,7 @@ object IO { checkCall(_LIB.mxDataIterGetIterInfo(handle, name, desc, argNames, argTypes, argDescs)) val paramStr = Base.ctypes2docstring(argNames, argTypes, argDescs) val docStr = s"${name.value}\n${desc.value}\n\n$paramStr\n" -// logger.debug(docStr) + logger.debug(docStr) return (name.value, creator(handle)) } @@ -37,14 +37,18 @@ object IO { checkCall(_LIB.mxDateIterCreateIter(handle, keys, vals, out)) return new MXDataIter(out) } - - def _init_data(data: List[NDArray], allowEmpty: Boolean, - defaultName: String): List[Tuple2[]] = { - - } } -abstract class DataIter (val batchSize: Int = 0) { +//class for batch of data +class DataBatch(val data: NDArray, + val label: NDArray, + val index: List[Long], + val pad: Int) + +/** + *DataIter object in mxnet. + */ +abstract class DataIter { /** * reset the iterator */ @@ -79,6 +83,14 @@ abstract class DataIter (val batchSize: Int = 0) { * @return */ def getIndex(): List[Long] + + /** + * return next batch of data + * @return + */ + def next(): DataBatch = { + return new DataBatch(getData(), getLabel(), getIndex(), getPad()) + } } /** diff --git a/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala b/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala index 553f99420f04..db58dca6dac2 100644 --- a/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala +++ b/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala @@ -1,13 +1,17 @@ package ml.dmlc.mxnet import org.scalatest.{BeforeAndAfterAll, FunSuite} +import scala.sys.process._ class IOSuite extends FunSuite with BeforeAndAfterAll { test("test MNISTIter") { + //mkdir data + "./get_data.sh" ! + val params = Map( - "image" -> "/home/hzx/workspace/git/mxnet-scala/mxnet/tests/python/unittest/data/train-images-idx3-ubyte", - "label" -> "/home/hzx/workspace/git/mxnet-scala/mxnet/tests/python/unittest/data/train-labels-idx1-ubyte", + "image" -> "data/train-images-idx3-ubyte", + "label" -> "data/train-labels-idx1-ubyte", "data_shape" -> "(784,)", "batch_size" -> "100", "shuffle" -> "1", @@ -18,33 +22,46 @@ class IOSuite extends FunSuite with BeforeAndAfterAll { // println("create MNISTIter") val mnist_iter = IO.iterCreateFuncs("MNISTIter")(params) mnist_iter.reset() + val nBatch = 600 + var batchCount = 0 while(mnist_iter.iterNext()) { - val data = mnist_iter.getData() - val label = mnist_iter.getLabel() - val index = mnist_iter.getIndex() - val pad = mnist_iter.getPad() + val batch = mnist_iter.next() + batchCount+=1 } + //test loop + assert(nBatch === batchCount) + //test reset + mnist_iter.reset() + mnist_iter.iterNext() + val label0 = mnist_iter.getLabel().toArray + mnist_iter.iterNext() + mnist_iter.iterNext() + mnist_iter.reset() + mnist_iter.iterNext() + val label1 = mnist_iter.getLabel().toArray + assert(label0 === label1) } - test("test ImageRecordIter") { - val params = Map( - "path_imgrec" -> "/home/hzx/workspace/git/mxnet-scala/mxnet/tests/python/unittest/data/cifar/train.rec", - "mean_img" -> "/home/hzx/workspace/git/mxnet-scala/mxnet/tests/python/unittest/data/cifar/cifar10_mean.bin", - "rand_crop" -> "false", - "and_mirror" -> "false", - "shuffle" -> "false", - "data_shape" -> "(3,28,28)", - "batch_size" -> "100", - "preprocess_threads" -> "4", - "prefetch_buffer" -> "1" - ) - val img_iter = IO.iterCreateFuncs("ImageRecordIter")(params) - img_iter.reset() - while(img_iter.iterNext()) { - val data = img_iter.getData() - val label = img_iter.getLabel() - val index = img_iter.getIndex() - val pad = img_iter.getPad() - } +// test("test ImageRecordIter") { +// val params = Map( +// "path_imgrec" -> "data/cifar/train.rec", +// "mean_img" -> "data/cifar/cifar10_mean.bin", +// "rand_crop" -> "false", +// "and_mirror" -> "false", +// "shuffle" -> "false", +// "data_shape" -> "(3,28,28)", +// "batch_size" -> "100", +// "preprocess_threads" -> "4", +// "prefetch_buffer" -> "1" +// ) +// val img_iter = IO.iterCreateFuncs("ImageRecordIter")(params) +// img_iter.reset() +// while(img_iter.iterNext()) { +// val batch = img_iter.next() +// } +// } + + test("test NDarryIter") { + } } diff --git a/scala-package/native/linux-x86_64/pom.xml b/scala-package/native/linux-x86_64/pom.xml index d8de358ca7d4..1accc8116bd3 100644 --- a/scala-package/native/linux-x86_64/pom.xml +++ b/scala-package/native/linux-x86_64/pom.xml @@ -67,6 +67,12 @@ + + + + + + @@ -112,11 +118,11 @@ -shared - - -fopenmp -lopencv_calib3d - + -fopenmp + ${ldflags.opencv} -Wl,--whole-archive - ../../../lib/libmxnet.a ../../../dmlc-core/libdmlc.a + + ../../../lib/libmxnet.a ../../../dmlc-core/libdmlc.a -Wl,--no-whole-archive From e01f6fa9e6ebb8749b3e7b621ae19db6b164cf6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E5=AD=90=E8=BD=A9?= Date: Mon, 28 Dec 2015 01:29:58 -0800 Subject: [PATCH 08/14] modify linux settings --- Makefile | 11 +++ scala-package/core/scripts/get_mnist_data.sh | 11 +++ .../src/main/scala/ml/dmlc/mxnet/IO.scala | 73 +++++++++++++----- .../test/scala/ml/dmlc/mxnet/IOSuite.scala | 75 ++++++++++++------- scala-package/native/linux-x86_64/pom.xml | 15 +++- 5 files changed, 134 insertions(+), 51 deletions(-) create mode 100755 scala-package/core/scripts/get_mnist_data.sh diff --git a/Makefile b/Makefile index b8f5408d3702..b01e61e939c6 100644 --- a/Makefile +++ b/Makefile @@ -93,6 +93,9 @@ endif all: lib/libmxnet.a lib/libmxnet.so $(BIN) +# test scala +scala: scala-package/native/linux-x86_64/target/libmxnet-scala.so + SRC = $(wildcard src/*.cc src/*/*.cc) OBJ = $(patsubst src/%.cc, build/%.o, $(SRC)) CUSRC = $(wildcard src/*/*.cu) @@ -147,6 +150,9 @@ lib/libmxnet.so: $(ALL_DEP) @mkdir -p $(@D) $(CXX) $(CFLAGS) -shared -o $@ $(filter %.o %.a, $^) $(LDFLAGS) +scala-package/native/linux-x86_64/target/libmxnet-scala.so: scala-package/native/src/main/native/jni_helper_func.h scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc $(LIB_DEP) lib/libmxnet.a + $(CXX) -std=c++0x $(CFLAGS) -Iscala-package/native/linux-x86_64/target/custom-javah -I${JAVA_HOME}/include -I${JAVA_HOME}/include/linux -shared -o $@ $(filter %.cpp %.o %.c %.cc, $^) -Wl,--whole-archive $(filter %.a, $^) -Wl,--no-whole-archive $(LDFLAGS) + # ps-lite $(PS_PATH)/build/libps.a: $(MAKE) CXX=$(CXX) DEPS_PATH=$(DEPS_PATH) -C $(PS_PATH) ps @@ -163,6 +169,11 @@ $(BIN) : include tests/cpp/unittest.mk +info: + echo $(CFLAGS) + echo $(LDFLAGS) + echo $(LIB_DEP) + test: $(TEST) lint: rcpplint diff --git a/scala-package/core/scripts/get_mnist_data.sh b/scala-package/core/scripts/get_mnist_data.sh new file mode 100755 index 000000000000..e080144f6663 --- /dev/null +++ b/scala-package/core/scripts/get_mnist_data.sh @@ -0,0 +1,11 @@ +data_path="./data" +if [ ! -d "$data_path" ]; then + mkdir -p "$data_path" +fi + +mnist_data_path="./data/mnist.zip" +if [ ! -f "$mnist_data_path" ]; then + wget http://webdocs.cs.ualberta.ca/~bx3/data/mnist.zip -P $data_path + cd $data_path + unzip -u mnist.zip +fi diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala index d293f2215000..c7f551f5add3 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala @@ -6,17 +6,32 @@ import org.slf4j.LoggerFactory import scala.collection.mutable.ListBuffer object IO { - private val logger = LoggerFactory.getLogger(classOf[DataIter]) type IterCreateFunc = (Map[String, String]) => DataIter - val iterCreateFuncs: Map[String, IterCreateFunc] = _initIOModule() - def _initIOModule(): Map[String, IterCreateFunc] = { + private val logger = LoggerFactory.getLogger(classOf[DataIter]) + private val iterCreateFuncs: Map[String, IterCreateFunc] = _initIOModule() + + /** + * create iterator via iterName and params + * @param iterName name of iterator; "MNISTIter" or "ImageRecordIter" + * @param params paramters for create iterator + * @return + */ + def createIterator(iterName: String, params: Map[String, String]): DataIter = { + return iterCreateFuncs(iterName)(params) + } + + /** + * initi all IO creator Functions + * @return + */ + private def _initIOModule(): Map[String, IterCreateFunc] = { val IterCreators = new ListBuffer[DataIterCreator] checkCall(_LIB.mxListDataIters(IterCreators)) IterCreators.map(_makeIOIterator).toMap } - def _makeIOIterator(handle: DataIterCreator): (String, IterCreateFunc) = { + private def _makeIOIterator(handle: DataIterCreator): (String, IterCreateFunc) = { val name = new RefString val desc = new RefString val argNames = new ListBuffer[String] @@ -25,11 +40,17 @@ object IO { checkCall(_LIB.mxDataIterGetIterInfo(handle, name, desc, argNames, argTypes, argDescs)) val paramStr = Base.ctypes2docstring(argNames, argTypes, argDescs) val docStr = s"${name.value}\n${desc.value}\n\n$paramStr\n" -// logger.debug(docStr) + logger.debug(docStr) return (name.value, creator(handle)) } - def creator(handle: DataIterCreator)( + /** + * + * @param handle + * @param params + * @return + */ + private def creator(handle: DataIterCreator)( params: Map[String, String]): DataIter = { val out = new DataIterHandle val keys = params.keys.toArray @@ -37,13 +58,24 @@ object IO { checkCall(_LIB.mxDateIterCreateIter(handle, keys, vals, out)) return new MXDataIter(out) } - - def _init_data(data: List[NDArray], allowEmpty: Boolean, - defaultName: String): List[Tuple2[]] = { - - } } +/** + * batch of data + * @param data + * @param label + * @param index + * @param pad + */ +class DataBatch(val data: NDArray, + val label: NDArray, + val index: List[Long], + val pad: Int) + +/** + * + * @param batchSize + */ abstract class DataIter (val batchSize: Int = 0) { /** * reset the iterator @@ -55,6 +87,14 @@ abstract class DataIter (val batchSize: Int = 0) { */ def iterNext(): Boolean + /** + * get next data batch from iterator + * @return + */ + def next(): DataBatch = { + return new DataBatch(getData(), getLabel(), getIndex(), getPad()) + } + /** * get data of current batch * @return the data of current batch @@ -149,16 +189,9 @@ class MXDataIter(var handle: DataIterHandle) extends DataIter { } /** - * NDArrayIter object in mxnet. Taking NDArray or numpy array to get dataiter. - * @param data a list of NDArray - * @param label a list of NDArray - * @param batch_size Batch Size - * @param shuffle Whether to shuffle the data - * @param last_batch_handle 'pad', 'discard' or 'roll_over', How to handle the last batch + * To do */ -class NDArrayIter(var data: List[NDArray], var label: List[NDArray], - var batch_size: Int, var shuffle: Boolean, - var last_batch_handle: String) extends DataIter { +class ArrayDataIter() extends DataIter { /** * reset the iterator */ diff --git a/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala b/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala index 553f99420f04..3f0a80747657 100644 --- a/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala +++ b/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala @@ -1,13 +1,17 @@ package ml.dmlc.mxnet import org.scalatest.{BeforeAndAfterAll, FunSuite} +import scala.sys.process._ class IOSuite extends FunSuite with BeforeAndAfterAll { test("test MNISTIter") { + //get data + "./scripts/get_mnist_data.sh" ! + val params = Map( - "image" -> "/home/hzx/workspace/git/mxnet-scala/mxnet/tests/python/unittest/data/train-images-idx3-ubyte", - "label" -> "/home/hzx/workspace/git/mxnet-scala/mxnet/tests/python/unittest/data/train-labels-idx1-ubyte", + "image" -> "data/train-images-idx3-ubyte", + "label" -> "data/train-labels-idx1-ubyte", "data_shape" -> "(784,)", "batch_size" -> "100", "shuffle" -> "1", @@ -15,36 +19,49 @@ class IOSuite extends FunSuite with BeforeAndAfterAll { "silent" -> "0", "seed" -> "10" ) -// println("create MNISTIter") - val mnist_iter = IO.iterCreateFuncs("MNISTIter")(params) + val nBatch = 600 + var batchCount = 0 + //println("create MNISTIter") + val mnist_iter = IO.createIterator("MNISTIter", params) + //test_loop mnist_iter.reset() while(mnist_iter.iterNext()) { - val data = mnist_iter.getData() - val label = mnist_iter.getLabel() - val index = mnist_iter.getIndex() - val pad = mnist_iter.getPad() + val batch = mnist_iter.next() + batchCount+=1 } + assert(nBatch === batchCount) + //test reset + mnist_iter.reset() + mnist_iter.iterNext() + val label0 = mnist_iter.getLabel().toArray + mnist_iter.iterNext() + mnist_iter.iterNext() + mnist_iter.iterNext() + mnist_iter.reset() + mnist_iter.iterNext() + val label1 = mnist_iter.getLabel().toArray + assert(label0 === label1) } - test("test ImageRecordIter") { - val params = Map( - "path_imgrec" -> "/home/hzx/workspace/git/mxnet-scala/mxnet/tests/python/unittest/data/cifar/train.rec", - "mean_img" -> "/home/hzx/workspace/git/mxnet-scala/mxnet/tests/python/unittest/data/cifar/cifar10_mean.bin", - "rand_crop" -> "false", - "and_mirror" -> "false", - "shuffle" -> "false", - "data_shape" -> "(3,28,28)", - "batch_size" -> "100", - "preprocess_threads" -> "4", - "prefetch_buffer" -> "1" - ) - val img_iter = IO.iterCreateFuncs("ImageRecordIter")(params) - img_iter.reset() - while(img_iter.iterNext()) { - val data = img_iter.getData() - val label = img_iter.getLabel() - val index = img_iter.getIndex() - val pad = img_iter.getPad() - } - } + /** + * link error + */ +// test("test ImageRecordIter") { +// val params = Map( +// "path_imgrec" -> "data/cifar/train.rec", +// "mean_img" -> "data/cifar/cifar10_mean.bin", +// "rand_crop" -> "false", +// "and_mirror" -> "false", +// "shuffle" -> "false", +// "data_shape" -> "(3,28,28)", +// "batch_size" -> "100", +// "preprocess_threads" -> "4", +// "prefetch_buffer" -> "1" +// ) +// val img_iter = IO.createIterator("ImageRecordIter", params) +// img_iter.reset() +// while(img_iter.iterNext()) { +// +// } +// } } diff --git a/scala-package/native/linux-x86_64/pom.xml b/scala-package/native/linux-x86_64/pom.xml index d38bf32ea2c4..428352073c98 100644 --- a/scala-package/native/linux-x86_64/pom.xml +++ b/scala-package/native/linux-x86_64/pom.xml @@ -13,6 +13,10 @@ MXNet Scala Package - Native Linux-x86_64 http://maven.apache.org + + opencv.pkg.txt + + so @@ -74,6 +78,12 @@ + + + + + + @@ -103,7 +113,7 @@ - -msse3 -funroll-loops -Wno-unused-parameter -Wno-unknown-pragmas + -msse3 -funroll-loops -Wno-unused-parameter -Wno-unknown-pragmas -fopenmp @@ -119,7 +129,8 @@ -shared - -fopenmp ${ldflags.blas} + -lm -lrt + ${ldflags.opencv} -Wl,--whole-archive ../../../lib/libmxnet.a ../../../dmlc-core/libdmlc.a From 84e00fde1214f15a7ac89230a13eb2a064808a0c Mon Sep 17 00:00:00 2001 From: yanqingmen Date: Mon, 28 Dec 2015 21:52:35 +0800 Subject: [PATCH 09/14] modify linux settings --- scala-package/core/get_data.sh | 9 --------- .../core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala | 13 ++++++++----- scala-package/native/linux-x86_64/pom.xml | 6 ++++-- tests/travis/run_test.sh | 6 ------ 4 files changed, 12 insertions(+), 22 deletions(-) delete mode 100755 scala-package/core/get_data.sh diff --git a/scala-package/core/get_data.sh b/scala-package/core/get_data.sh deleted file mode 100755 index fa10b31ba5ac..000000000000 --- a/scala-package/core/get_data.sh +++ /dev/null @@ -1,9 +0,0 @@ -if [ ! -d "./data" ]; then - mkdir -p ./data -fi -if [ ! -d "./data/mnist.zip" ]; then - wget http://webdocs.cs.ualberta.ca/~bx3/data/mnist.zip -P data/ -fi -cd data -rm *-ubyte -unzip -u mnist.zip \ No newline at end of file diff --git a/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala b/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala index 9a3c96092647..d93cecdc766a 100644 --- a/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala +++ b/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala @@ -47,21 +47,24 @@ class IOSuite extends FunSuite with BeforeAndAfterAll { /** - * link error + * not work now */ // test("test ImageRecordIter") { +// //get data +// "./scripts/get_cifar_data.sh" ! +// // val params = Map( // "path_imgrec" -> "data/cifar/train.rec", // "mean_img" -> "data/cifar/cifar10_mean.bin", -// "rand_crop" -> "false", -// "and_mirror" -> "false", -// "shuffle" -> "false", +// "rand_crop" -> "False", +// "and_mirror" -> "False", +// "shuffle" -> "False", // "data_shape" -> "(3,28,28)", // "batch_size" -> "100", // "preprocess_threads" -> "4", // "prefetch_buffer" -> "1" // ) -// val img_iter = IO.iterCreateFuncs("ImageRecordIter")(params) +// val img_iter = IO.createIterator("ImageRecordIter", params) // img_iter.reset() // while(img_iter.iterNext()) { // val batch = img_iter.next() diff --git a/scala-package/native/linux-x86_64/pom.xml b/scala-package/native/linux-x86_64/pom.xml index 428352073c98..a0dedfb8122b 100644 --- a/scala-package/native/linux-x86_64/pom.xml +++ b/scala-package/native/linux-x86_64/pom.xml @@ -124,17 +124,19 @@ -DMSHADOW_USE_CBLAS=${use.cblas} -DMSHADOW_USE_MKL=${use.mkl} -fPIC + -shared - -lm -lrt - ${ldflags.opencv} -Wl,--whole-archive ../../../lib/libmxnet.a ../../../dmlc-core/libdmlc.a -Wl,--no-whole-archive + -lm -lrt -fopenmp + ${ldflags.opencv} + ${ldflags.blas} diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index f3919a06fa20..43ed4cc922a0 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -131,12 +131,6 @@ if [ ${TASK} == "scala_test" ]; then mvn integration-test -P osx-x86_64 --log-file scala_test_results.txt fi if [ ${TRAVIS_OS_NAME} == "linux" ]; then - # (Yizhi Liu) I'm not sure it is a proper solution, - # which is mentioned here: - # http://stackoverflow.com/questions/9558909/jni-symbol-lookup-error-in-shared-library-on-linux/13086028#13086028 - # I really don't know why we have to export LD_PRELOAD - # to make libblas loaded in travis. It just works. - export LD_PRELOAD=/usr/lib/libblas/libblas.so # use g++-4.8 for linux mvn clean package -P linux-x86_64 -D cxx=g++-4.8 -D ldflags.blas=-lblas mvn integration-test -P linux-x86_64 -D cxx=g++-4.8 -D ldflags.blas=-lblas --log-file scala_test_results.txt From 8449c892ca2e51bfdf6d3ca2f6147d92e77bcce9 Mon Sep 17 00:00:00 2001 From: yanqingmen Date: Mon, 28 Dec 2015 21:56:41 +0800 Subject: [PATCH 10/14] recover Makefile --- Makefile | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index b01e61e939c6..d7448a2be43d 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ CFLAGS = -DMSHADOW_FORCE_STREAM $(WARNFLAGS) # CFLAGS for debug ifeq ($(DEBUG), 1) - CFLAGS += -g -O0 + CFLAGS += -g -O0 -DDMLC_LOG_FATAL_THROW=0 else CFLAGS += -O3 endif @@ -93,9 +93,6 @@ endif all: lib/libmxnet.a lib/libmxnet.so $(BIN) -# test scala -scala: scala-package/native/linux-x86_64/target/libmxnet-scala.so - SRC = $(wildcard src/*.cc src/*/*.cc) OBJ = $(patsubst src/%.cc, build/%.o, $(SRC)) CUSRC = $(wildcard src/*/*.cu) @@ -150,9 +147,6 @@ lib/libmxnet.so: $(ALL_DEP) @mkdir -p $(@D) $(CXX) $(CFLAGS) -shared -o $@ $(filter %.o %.a, $^) $(LDFLAGS) -scala-package/native/linux-x86_64/target/libmxnet-scala.so: scala-package/native/src/main/native/jni_helper_func.h scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc $(LIB_DEP) lib/libmxnet.a - $(CXX) -std=c++0x $(CFLAGS) -Iscala-package/native/linux-x86_64/target/custom-javah -I${JAVA_HOME}/include -I${JAVA_HOME}/include/linux -shared -o $@ $(filter %.cpp %.o %.c %.cc, $^) -Wl,--whole-archive $(filter %.a, $^) -Wl,--no-whole-archive $(LDFLAGS) - # ps-lite $(PS_PATH)/build/libps.a: $(MAKE) CXX=$(CXX) DEPS_PATH=$(DEPS_PATH) -C $(PS_PATH) ps @@ -169,11 +163,6 @@ $(BIN) : include tests/cpp/unittest.mk -info: - echo $(CFLAGS) - echo $(LDFLAGS) - echo $(LIB_DEP) - test: $(TEST) lint: rcpplint @@ -211,4 +200,4 @@ clean_all: clean cd $(PS_PATH); make clean; cd - -include build/*.d --include build/*/*.d +-include build/*/*.d \ No newline at end of file From 19f445e332195a29cd9abe8e1075fea3e5bdb25e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E5=AD=90=E8=BD=A9?= Date: Tue, 29 Dec 2015 00:18:37 -0800 Subject: [PATCH 11/14] rm debug codes and fix typo --- .../src/main/scala/ml/dmlc/mxnet/IO.scala | 6 +++- .../test/scala/ml/dmlc/mxnet/IOSuite.scala | 27 ++++++++-------- .../main/native/ml_dmlc_mxnet_native_c_api.cc | 31 ++++++++++--------- 3 files changed, 34 insertions(+), 30 deletions(-) diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala index b1a0cc963bd5..227b7e3fe7fd 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala @@ -126,9 +126,13 @@ abstract class DataIter (val batchSize: Int = 0) { * DataIter built in MXNet. * @param handle the handle to the underlying C++ Data Iterator */ -class MXDataIter(var handle: DataIterHandle) extends DataIter { +class MXDataIter(val handle: DataIterHandle) extends DataIter { private val logger = LoggerFactory.getLogger(classOf[MXDataIter]) + override def finalize() = { + checkCall(_LIB.mxDataIterFree(handle)) + } + /** * reset the iterator */ diff --git a/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala b/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala index d93cecdc766a..dcc4747f8866 100644 --- a/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala +++ b/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala @@ -20,28 +20,27 @@ class IOSuite extends FunSuite with BeforeAndAfterAll { "seed" -> "10" ) - //println("create MNISTIter") - val mnist_iter = IO.createIterator("MNISTIter", params) + val mnistIter = IO.createIterator("MNISTIter", params) //test_loop - mnist_iter.reset() + mnistIter.reset() val nBatch = 600 var batchCount = 0 - while(mnist_iter.iterNext()) { - val batch = mnist_iter.next() + while(mnistIter.iterNext()) { + val batch = mnistIter.next() batchCount+=1 } //test loop assert(nBatch === batchCount) //test reset - mnist_iter.reset() - mnist_iter.iterNext() - val label0 = mnist_iter.getLabel().toArray - mnist_iter.iterNext() - mnist_iter.iterNext() - mnist_iter.iterNext() - mnist_iter.reset() - mnist_iter.iterNext() - val label1 = mnist_iter.getLabel().toArray + mnistIter.reset() + mnistIter.iterNext() + val label0 = mnistIter.getLabel().toArray + mnistIter.iterNext() + mnistIter.iterNext() + mnistIter.iterNext() + mnistIter.reset() + mnistIter.iterNext() + val label1 = mnistIter.getLabel().toArray assert(label0 === label1) } diff --git a/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc b/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc index 4a1412017917..9d9502aca78c 100644 --- a/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc +++ b/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc @@ -522,19 +522,21 @@ JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxDateIterCreateIter char** keys = new char*[paramSize]; char** vals = new char*[paramSize]; jstring jkey, jval; + //use strcpy and release char* created by JNI inplace for(int i=0; iGetObjectArrayElement(jkeys, i); - keys[i] = (char*)env->GetStringUTFChars(jkey, 0); + const char* ckey = env->GetStringUTFChars(jkey, 0); + keys[i] = new char[env->GetStringLength(jkey)]; + strcpy(keys[i], ckey); + env->ReleaseStringUTFChars(jkey, ckey); + jval = (jstring) env->GetObjectArrayElement(jvals, i); - vals[i] = (char*)env->GetStringUTFChars(jval, 0); + const char* cval = env->GetStringUTFChars(jval, 0); + vals[i] = new char[env->GetStringLength(jval)]; + strcpy(vals[i], cval); + env->ReleaseStringUTFChars(jval, cval); } -// printf("paramSize: %d\n", paramSize); -// for(int i=0; iGetFieldID(hClass, "value", "J"); env->SetLongField(dataIterHandle, ptr, (long)out); - //release const char* + //release keys and vals for(int i=0; iGetObjectArrayElement(jkeys, i); - env->ReleaseStringUTFChars(jkey,(const char*)keys[i]); - jval = (jstring) env->GetObjectArrayElement(jvals, i); - env->ReleaseStringUTFChars(jval,(const char*)vals[i]); + delete[] keys[i]; + delete[] vals[i]; } + delete[] keys; + delete[] vals; + return ret; } @@ -649,9 +652,7 @@ JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxDataIterGetIndex jmethodID listAppend = env->GetMethodID(listClass, "$plus$eq", "(Ljava/lang/Object;)Lscala/collection/mutable/ListBuffer;"); -// printf("outSize: %ld\n", coutSize); for(int i=0; iCallObjectMethod(outIndex, listAppend, (jlong)coutIndex[i]); } return ret; From 1f239ee1fc5d14d142c0d9061cffec4342eb5400 Mon Sep 17 00:00:00 2001 From: yanqingmen Date: Tue, 29 Dec 2015 21:23:05 +0800 Subject: [PATCH 12/14] small change --- .../src/main/scala/ml/dmlc/mxnet/IO.scala | 10 +- .../main/scala/ml/dmlc/mxnet/LibInfo.scala | 2 +- scala-package/scala_test_results.txt | 664 ++++++++++++++++++ 3 files changed, 670 insertions(+), 6 deletions(-) create mode 100644 scala-package/scala_test_results.txt diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala index b1a0cc963bd5..dd67f9fcf565 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala @@ -55,7 +55,7 @@ object IO { val out = new DataIterHandle val keys = params.keys.toArray val vals = params.values.toArray - checkCall(_LIB.mxDateIterCreateIter(handle, keys, vals, out)) + checkCall(_LIB.mxDataIterCreateIter(handle, keys, vals, out)) return new MXDataIter(out) } } @@ -68,10 +68,10 @@ object IO { * @param index * @param pad */ -class DataBatch(val data: NDArray, - val label: NDArray, - val index: List[Long], - val pad: Int) +case class DataBatch(val data: NDArray, + val label: NDArray, + val index: List[Long], + val pad: Int) /** *DataIter object in mxnet. diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala index 0840093d86cd..cdc69d857c8f 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala @@ -87,7 +87,7 @@ class LibInfo { //DataIter Funcs @native def mxListDataIters(handles: ListBuffer[DataIterCreator]): Int - @native def mxDateIterCreateIter(handle: DataIterCreator, + @native def mxDataIterCreateIter(handle: DataIterCreator, keys: Array[String], vals: Array[String], out: DataIterHandle): Int diff --git a/scala-package/scala_test_results.txt b/scala-package/scala_test_results.txt new file mode 100644 index 000000000000..fad7ea59cbae --- /dev/null +++ b/scala-package/scala_test_results.txt @@ -0,0 +1,664 @@ +[INFO] Scanning for projects... +[WARNING] +[WARNING] Some problems were encountered while building the effective model for ml.dmlc.mxnet:mxnet-scala-core:jar:0.1-SNAPSHOT +[WARNING] 'build.plugins.plugin.version' for org.apache.maven.plugins:maven-compiler-plugin is missing. @ line 42, column 15 +[WARNING] +[WARNING] Some problems were encountered while building the effective model for ml.dmlc.mxnet:libmxnet-scala:so:0.0.1-SNAPSHOT +[WARNING] 'build.plugins.plugin.version' for org.apache.maven.plugins:maven-compiler-plugin is missing. @ line 34, column 15 +[WARNING] +[WARNING] It is highly recommended to fix these problems because they threaten the stability of your build. +[WARNING] +[WARNING] For this reason, future Maven versions might no longer support building such malformed projects. +[WARNING] +[INFO] ------------------------------------------------------------------------ +[INFO] Reactor Build Order: +[INFO] +[INFO] MXNet Scala Package - Parent +[INFO] MXNet Scala Package - Core +[INFO] MXNet Scala Package - Native Parent +[INFO] MXNet Scala Package - Native Linux-x86_64 +[INFO] +[INFO] ------------------------------------------------------------------------ +[INFO] Building MXNet Scala Package - Parent 0.1-SNAPSHOT +[INFO] ------------------------------------------------------------------------ +[INFO] +[INFO] ------------------------------------------------------------------------ +[INFO] Building MXNet Scala Package - Core 0.1-SNAPSHOT +[INFO] ------------------------------------------------------------------------ +[INFO] +[INFO] --- maven-resources-plugin:2.7:copy-resources (copy-resources) @ mxnet-scala-core --- +[WARNING] File encoding has not been set, using platform encoding UTF-8, i.e. build is platform dependent! +[WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! +[INFO] Copying 2 resources +[INFO] +[INFO] --- maven-resources-plugin:2.7:resources (default-resources) @ mxnet-scala-core --- +[WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! +[INFO] Copying 2 resources +[INFO] +[INFO] --- scala-maven-plugin:3.2.2:compile (default) @ mxnet-scala-core --- +[INFO] Nothing to compile - all classes are up to date +[INFO] +[INFO] --- maven-compiler-plugin:2.0.2:compile (default-compile) @ mxnet-scala-core --- +[INFO] No sources to compile +[INFO] +[INFO] --- scala-maven-plugin:3.2.2:compile (compile) @ mxnet-scala-core --- +[INFO] Nothing to compile - all classes are up to date +[INFO] +[INFO] --- maven-resources-plugin:2.7:testResources (default-testResources) @ mxnet-scala-core --- +[WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! +[INFO] skip non existing resourceDirectory /home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/core/src/test/resources +[INFO] +[INFO] --- maven-compiler-plugin:2.0.2:testCompile (default-testCompile) @ mxnet-scala-core --- +[INFO] No sources to compile +[INFO] +[INFO] --- scala-maven-plugin:3.2.2:testCompile (test-compile) @ mxnet-scala-core --- +[INFO] Nothing to compile - all classes are up to date +[INFO] +[INFO] --- maven-surefire-plugin:2.7:test (default-test) @ mxnet-scala-core --- +[INFO] Tests are skipped. +[INFO] +[INFO] --- maven-jar-plugin:2.5:jar (default-jar) @ mxnet-scala-core --- +[INFO] Building jar: /home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/core/target/mxnet-scala-core-0.1-SNAPSHOT.jar +[INFO] +[INFO] --- maven-dependency-plugin:2.9:copy-dependencies (copy-dependencies) @ mxnet-scala-core --- +[INFO] log4j:log4j:jar:1.2.17 already exists in destination. +[INFO] org.slf4j:slf4j-api:jar:1.7.5 already exists in destination. +[INFO] org.slf4j:slf4j-log4j12:jar:1.7.7 already exists in destination. +[INFO] org.scala-lang:scala-library:jar:2.10.4 already exists in destination. +[INFO] commons-codec:commons-codec:jar:1.10 already exists in destination. +[INFO] +[INFO] --- maven-jar-plugin:2.5:jar (default) @ mxnet-scala-core --- +[INFO] +[INFO] --- scalatest-maven-plugin:1.0:test (test) @ mxnet-scala-core --- +Discovery starting. +Discovery completed in 183 milliseconds. +Run starting. Expected test count is: 19 +KVStoreSuite: +2015-12-28 23:08:03,245 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +_set_value + + +Parameters +---------- +src : real_t +Source input to the function. + + +2015-12-28 23:08:03,245 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +_plus + + +Parameters +---------- +lhs : NDArray +Left operand to the function. +rhs : NDArray +Right operand to the function. + + +2015-12-28 23:08:03,246 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +_minus + + +Parameters +---------- +lhs : NDArray +Left operand to the function. +rhs : NDArray +Right operand to the function. + + +2015-12-28 23:08:03,246 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +_mul + + +Parameters +---------- +lhs : NDArray +Left operand to the function. +rhs : NDArray +Right operand to the function. + + +2015-12-28 23:08:03,247 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +_div + + +Parameters +---------- +lhs : NDArray +Left operand to the function. +rhs : NDArray +Right operand to the function. + + +2015-12-28 23:08:03,247 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +dot +Calcuate 2D matrix multiplication + +Parameters +---------- +lhs : NDArray +Left operand to the function. +rhs : NDArray +Right operand to the function. + + +2015-12-28 23:08:03,247 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +_onehot_encode + + +Parameters +---------- +lhs : NDArray +Left operand to the function. +rhs : NDArray +Right operand to the function. + + +2015-12-28 23:08:03,248 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +choose_element_0index +Choose one element from each line(row for python, column for R/Julia) in lhs according to index indicated by rhs. This function assume rhs uses 0-based index. + +Parameters +---------- +lhs : NDArray +Left operand to the function. +rhs : NDArray +Right operand to the function. + + +2015-12-28 23:08:03,248 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +_plus_scalar + + +Parameters +---------- +lhs : NDArray +Left operand to the function. +rhs : real_t +Right operand to the function. + + +2015-12-28 23:08:03,249 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +_minus_scalar + + +Parameters +---------- +lhs : NDArray +Left operand to the function. +rhs : real_t +Right operand to the function. + + +2015-12-28 23:08:03,249 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +_mul_scalar + + +Parameters +---------- +lhs : NDArray +Left operand to the function. +rhs : real_t +Right operand to the function. + + +2015-12-28 23:08:03,249 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +_div_scalar + + +Parameters +---------- +lhs : NDArray +Left operand to the function. +rhs : real_t +Right operand to the function. + + +2015-12-28 23:08:03,250 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +_rminus_scalar + + +Parameters +---------- +lhs : NDArray +Left operand to the function. +rhs : real_t +Right operand to the function. + + +2015-12-28 23:08:03,250 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +_rdiv_scalar + + +Parameters +---------- +lhs : NDArray +Left operand to the function. +rhs : real_t +Right operand to the function. + + +2015-12-28 23:08:03,250 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +_copyto + + +Parameters +---------- +src : NDArray +Source input to the function. + + +2015-12-28 23:08:03,251 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +_random_uniform + + +Parameters +---------- + + + +2015-12-28 23:08:03,251 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +_random_gaussian + + +Parameters +---------- + + + +2015-12-28 23:08:03,251 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +clip +Clip ndarray elements to range (a_min, a_max) + +Parameters +---------- +src : NDArray +Source input +a_min : real_t +Minimum value +a_max : real_t +Maximum value + + +2015-12-28 23:08:03,252 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +abs +Take absolute value of the src + +Parameters +---------- +src : NDArray +Source input to the function + + +2015-12-28 23:08:03,252 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +sign +Take sign value of the src + +Parameters +---------- +src : NDArray +Source input to the function + + +2015-12-28 23:08:03,252 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +round +Take round value of the src + +Parameters +---------- +src : NDArray +Source input to the function + + +2015-12-28 23:08:03,253 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +ceil +Take ceil value of the src + +Parameters +---------- +src : NDArray +Source input to the function + + +2015-12-28 23:08:03,253 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +floor +Take floor value of the src + +Parameters +---------- +src : NDArray +Source input to the function + + +2015-12-28 23:08:03,253 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +square +Take square of the src + +Parameters +---------- +src : NDArray +Source input to the function + + +2015-12-28 23:08:03,253 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +sqrt +Take sqrt of the src + +Parameters +---------- +src : NDArray +Source input to the function + + +2015-12-28 23:08:03,254 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +rsqrt +Take rsqrt of the src + +Parameters +---------- +src : NDArray +Source input to the function + + +2015-12-28 23:08:03,254 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +exp +Take exp of the src + +Parameters +---------- +src : NDArray +Source input to the function + + +2015-12-28 23:08:03,254 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +log +Take log of the src + +Parameters +---------- +src : NDArray +Source input to the function + + +2015-12-28 23:08:03,255 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +cos +Take cos of the src + +Parameters +---------- +src : NDArray +Source input to the function + + +2015-12-28 23:08:03,255 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +sin +Take sin of the src + +Parameters +---------- +src : NDArray +Source input to the function + + +2015-12-28 23:08:03,255 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +norm +Take L2 norm of the src.The result will be ndarray of shape (1,) on the same device. + +Parameters +---------- +src : NDArray +Source input to the function + + +2015-12-28 23:08:03,256 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +max +Take max of the src.The result will be ndarray of shape (1,) on the same device. + +Parameters +---------- +src : NDArray +Source input to the function + + +2015-12-28 23:08:03,256 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +min +Take min of the src.The result will be ndarray of shape (1,) on the same device. + +Parameters +---------- +src : NDArray +Source input to the function + + +2015-12-28 23:08:03,256 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: +sum +Take sum of the src.The result will be ndarray of shape (1,) on the same device. + +Parameters +---------- +src : NDArray +Source input to the function + + +- init and pull +- push and pull +update on key 3 +- updater runs when push +- get type +- get numWorkers and rank +2015-12-28 23:08:03,303 [ScalaTest-main-running-IOSuite] [ml.dmlc.mxnet.DataIter] [DEBUG] - ImageRecordIter +Create iterator for dataset packed in recordio. + +Parameters +---------- +path_imglist : string, optional, default='' +Dataset Param: Path to image list. +path_imgrec : string, optional, default='./data/imgrec.rec' +Dataset Param: Path to image record file. +label_width : int, optional, default='1' +Dataset Param: How many labels for an image. +data_shape : Shape(tuple), required +Dataset Param: Shape of each instance generated by the DataIter. +preprocess_threads : int, optional, default='4' +Backend Param: Number of thread to do preprocessing. +verbose : boolean, optional, default=True +Auxiliary Param: Whether to output parser information. +num_parts : int, optional, default='1' +partition the data into multiple parts +part_index : int, optional, default='0' +the index of the part will read +shuffle : boolean, optional, default=False +Augmentation Param: Whether to shuffle data. +seed : int, optional, default='0' +Augmentation Param: Random Seed. +verbose : boolean, optional, default=True +Auxiliary Param: Whether to output information. +batch_size : int (non-negative), required +Batch Param: Batch size. +round_batch : boolean, optional, default=True +Batch Param: Use round robin to handle overflow batch. +prefetch_buffer : long (non-negative), optional, default=4 +Backend Param: Number of prefetched parameters +rand_crop : boolean, optional, default=False +Augmentation Param: Whether to random crop on the image +crop_y_start : int, optional, default='-1' +Augmentation Param: Where to nonrandom crop on y. +crop_x_start : int, optional, default='-1' +Augmentation Param: Where to nonrandom crop on x. +max_rotate_angle : int, optional, default='0' +Augmentation Param: rotated randomly in [-max_rotate_angle, max_rotate_angle]. +max_aspect_ratio : float, optional, default=0 +Augmentation Param: denotes the max ratio of random aspect ratio augmentation. +max_shear_ratio : float, optional, default=0 +Augmentation Param: denotes the max random shearing ratio. +max_crop_size : int, optional, default='-1' +Augmentation Param: Maximum crop size. +min_crop_size : int, optional, default='-1' +Augmentation Param: Minimum crop size. +max_random_scale : float, optional, default=1 +Augmentation Param: Maxmum scale ratio. +min_random_scale : float, optional, default=1 +Augmentation Param: Minimum scale ratio. +max_img_size : float, optional, default=1e+10 +Augmentation Param: Maxmum image size after resizing. +min_img_size : float, optional, default=0 +Augmentation Param: Minimum image size after resizing. +rotate : int, optional, default='-1' +Augmentation Param: Rotate angle. +fill_value : int, optional, default='255' +Augmentation Param: Maximum value of illumination variation. +data_shape : Shape(tuple), required +Dataset Param: Shape of each instance generated by the DataIter. +seed : int, optional, default='0' +Augmentation Param: Random Seed. +mirror : boolean, optional, default=False +Augmentation Param: Whether to mirror the image. +rand_mirror : boolean, optional, default=False +Augmentation Param: Whether to mirror the image randomly. +mean_img : string, optional, default='' +Augmentation Param: Mean Image to be subtracted. +mean_r : float, optional, default=0 +Augmentation Param: Mean value on R channel. +mean_g : float, optional, default=0 +Augmentation Param: Mean value on G channel. +mean_b : float, optional, default=0 +Augmentation Param: Mean value on B channel. +mean_a : float, optional, default=0 +Augmentation Param: Mean value on Alpha channel. +scale : float, optional, default=1 +Augmentation Param: Scale in color space. +max_random_contrast : float, optional, default=0 +Augmentation Param: Maximum ratio of contrast variation. +max_random_illumination : float, optional, default=0 +Augmentation Param: Maximum value of illumination variation. +verbose : boolean, optional, default=True +Augmentation Param: Whether to print augmentor info. + + +2015-12-28 23:08:03,304 [ScalaTest-main-running-IOSuite] [ml.dmlc.mxnet.DataIter] [DEBUG] - MNISTIter +Create iterator for MNIST hand-written digit number recognition dataset. + +Parameters +---------- +image : string, optional, default='./train-images-idx3-ubyte' +Dataset Param: Mnist image path. +label : string, optional, default='./train-labels-idx1-ubyte' +Dataset Param: Mnist label path. +batch_size : int, optional, default='128' +Batch Param: Batch Size. +shuffle : boolean, optional, default=True +Augmentation Param: Whether to shuffle data. +flat : boolean, optional, default=False +Augmentation Param: Whether to flat the data into 1D. +seed : int, optional, default='0' +Augmentation Param: Random Seed. +silent : boolean, optional, default=False +Auxiliary Param: Whether to print out data info. +num_parts : int, optional, default='1' +partition the data into multiple parts +part_index : int, optional, default='0' +the index of the part will read +prefetch_buffer : long (non-negative), optional, default=4 +Backend Param: Number of prefetched parameters + + +2015-12-28 23:08:03,305 [ScalaTest-main-running-IOSuite] [ml.dmlc.mxnet.DataIter] [DEBUG] - CSVIter +Create iterator for dataset in csv. + +Parameters +---------- +data_csv : string, required +Dataset Param: Data csv path. +data_shape : Shape(tuple), required +Dataset Param: Shape of the data. +label_csv : string, optional, default='NULL' +Dataset Param: Label csv path. If is NULL, all labels will be returned as 0 +label_shape : Shape(tuple), optional, default=(1,) +Dataset Param: Shape of the label. + + +IOSuite: +[23:08:04] src/io/iter_mnist.cc:91: MNISTIter: load 60000 images, shuffle=1, shape=(100,784) +- test MNISTIter +NDArraySuite: +- to java array +- to scalar +- call toScalar on an ndarray which is not a scalar +- size and shape +- set scalar value +- copy from java array +- plus +- minus +- multiplication +- division +- clip +- sqrt +- norm +Run completed in 1 second, 908 milliseconds. +Total number of tests run: 19 +Suites: completed 4, aborted 0 +Tests: succeeded 19, failed 0, canceled 0, ignored 0, pending 0 +All tests passed. +[INFO] +[INFO] ------------------------------------------------------------------------ +[INFO] Building MXNet Scala Package - Native Parent 0.1-SNAPSHOT +[INFO] ------------------------------------------------------------------------ +[INFO] +[INFO] ------------------------------------------------------------------------ +[INFO] Building MXNet Scala Package - Native Linux-x86_64 0.0.1-SNAPSHOT +[INFO] ------------------------------------------------------------------------ +[INFO] +[INFO] --- maven-resources-plugin:2.7:copy-resources (copy-resources) @ libmxnet-scala --- +[WARNING] File encoding has not been set, using platform encoding UTF-8, i.e. build is platform dependent! +[WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! +[INFO] skip non existing resourceDirectory /home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/native/linux-x86_64/src/main/resources +[INFO] +[INFO] --- native-maven-plugin:1.0-alpha-7:initialize (default-initialize) @ libmxnet-scala --- +[INFO] +[INFO] --- native-maven-plugin:1.0-alpha-7:unzipinc (default-unzipinc) @ libmxnet-scala --- +[INFO] +[INFO] --- native-maven-plugin:1.0-alpha-7:javah (default-javah) @ libmxnet-scala --- +[INFO] +[INFO] --- maven-antrun-plugin:1.7:run (default) @ libmxnet-scala --- +[WARNING] Parameter tasks is deprecated, use target instead +[INFO] Executing tasks + +main: + [delete] Deleting: /home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/native/linux-x86_64/opencv.pkg.txt +[INFO] Executed tasks +[INFO] +[INFO] --- native-maven-plugin:1.0-alpha-7:javah (javah) @ libmxnet-scala --- +[INFO] /bin/sh -c cd /home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/native/linux-x86_64 && javah -o /home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/native/linux-x86_64/target/custom-javah/ml_dmlc_mxnet_native_c_api.h -classpath /home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/native/linux-x86_64/target/classes:/home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/core/target/mxnet-scala-core-0.1-SNAPSHOT.jar:/home/hzx/.m2/repository/org/scala-lang/scala-library/2.10.4/scala-library-2.10.4.jar:/home/hzx/.m2/repository/commons-codec/commons-codec/1.10/commons-codec-1.10.jar:/home/hzx/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/home/hzx/.m2/repository/org/slf4j/slf4j-api/1.7.5/slf4j-api-1.7.5.jar:/home/hzx/.m2/repository/org/slf4j/slf4j-log4j12/1.7.7/slf4j-log4j12-1.7.7.jar ml.dmlc.mxnet.LibInfo +[INFO] +[INFO] --- native-maven-plugin:1.0-alpha-7:compile (default-compile) @ libmxnet-scala --- +[INFO] +[INFO] --- native-maven-plugin:1.0-alpha-7:link (default-link) @ libmxnet-scala --- +[INFO] /bin/sh -c cd /home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/native/linux-x86_64 && g++-4.8 -shared -o/home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/native/linux-x86_64/target/libmxnet-scala.so target/objs/ml_dmlc_mxnet_native_c_api.o -Wl,--whole-archive ../../../lib/libmxnet.a ../../../dmlc-core/libdmlc.a -Wl,--no-whole-archive -lm -lrt -fopenmp /usr/lib/x86_64-linux-gnu/libopencv_calib3d.so /usr/lib/x86_64-linux-gnu/libopencv_contrib.so /usr/lib/x86_64-linux-gnu/libopencv_core.so /usr/lib/x86_64-linux-gnu/libopencv_features2d.so /usr/lib/x86_64-linux-gnu/libopencv_flann.so /usr/lib/x86_64-linux-gnu/libopencv_gpu.so /usr/lib/x86_64-linux-gnu/libopencv_highgui.so /usr/lib/x86_64-linux-gnu/libopencv_imgproc.so /usr/lib/x86_64-linux-gnu/libopencv_legacy.so /usr/lib/x86_64-linux-gnu/libopencv_ml.so /usr/lib/x86_64-linux-gnu/libopencv_objdetect.so /usr/lib/x86_64-linux-gnu/libopencv_ocl.so /usr/lib/x86_64-linux-gnu/libopencv_photo.so /usr/lib/x86_64-linux-gnu/libopencv_stitching.so /usr/lib/x86_64-linux-gnu/libopencv_superres.so /usr/lib/x86_64-linux-gnu/libopencv_ts.so /usr/lib/x86_64-linux-gnu/libopencv_video.so /usr/lib/x86_64-linux-gnu/libopencv_videostab.so -lopencv_calib3d -lopencv_contrib -lopencv_core -lopencv_features2d -lopencv_flann -lopencv_gpu -lopencv_highgui -lopencv_imgproc -lopencv_legacy -lopencv_ml -lopencv_objdetect -lopencv_ocl -lopencv_photo -lopencv_stitching -lopencv_superres -lopencv_ts -lopencv_video -lopencv_videostab -lblas +[INFO] +[INFO] --- maven-resources-plugin:2.7:testResources (default-testResources) @ libmxnet-scala --- +[WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! +[INFO] skip non existing resourceDirectory /home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/native/linux-x86_64/src/test/resources +[INFO] +[INFO] --- maven-compiler-plugin:3.3:testCompile (default-testCompile) @ libmxnet-scala --- +[INFO] No sources to compile +[INFO] +[INFO] --- maven-surefire-plugin:2.7:test (default-test) @ libmxnet-scala --- +[INFO] Tests are skipped. +[INFO] +[INFO] --- native-maven-plugin:1.0-alpha-7:inczip (default-inczip) @ libmxnet-scala --- +[INFO] ------------------------------------------------------------------------ +[INFO] Reactor Summary: +[INFO] +[INFO] MXNet Scala Package - Parent ...................... SUCCESS [0.033s] +[INFO] MXNet Scala Package - Core ........................ SUCCESS [4.727s] +[INFO] MXNet Scala Package - Native Parent ............... SUCCESS [0.001s] +[INFO] MXNet Scala Package - Native Linux-x86_64 ......... SUCCESS [2.198s] +[INFO] ------------------------------------------------------------------------ +[INFO] BUILD SUCCESS +[INFO] ------------------------------------------------------------------------ +[INFO] Total time: 7.507s +[INFO] Finished at: Mon Dec 28 23:08:07 CST 2015 +[INFO] Final Memory: 24M/341M +[INFO] ------------------------------------------------------------------------ From 3a017c5df62e7ba266b4a0adc3aa89057431aa52 Mon Sep 17 00:00:00 2001 From: yanqingmen Date: Tue, 29 Dec 2015 21:56:21 +0800 Subject: [PATCH 13/14] smll change --- scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala | 2 +- .../native/src/main/native/ml_dmlc_mxnet_native_c_api.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala b/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala index dcc4747f8866..fc6940cec50b 100644 --- a/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala +++ b/scala-package/core/src/test/scala/ml/dmlc/mxnet/IOSuite.scala @@ -50,7 +50,7 @@ class IOSuite extends FunSuite with BeforeAndAfterAll { */ // test("test ImageRecordIter") { // //get data -// "./scripts/get_cifar_data.sh" ! +// //"./scripts/get_cifar_data.sh" ! // // val params = Map( // "path_imgrec" -> "data/cifar/train.rec", diff --git a/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc b/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc index 9d9502aca78c..ade79168a9a7 100644 --- a/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc +++ b/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc @@ -514,7 +514,7 @@ JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxListDataIters return ret; } -JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxDateIterCreateIter +JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxDataIterCreateIter (JNIEnv * env, jobject obj, jobject creator, jobjectArray jkeys, jobjectArray jvals, jobject dataIterHandle) { //keys and values From d91c39cc78964fa90f0cd3c598d89d31284d6493 Mon Sep 17 00:00:00 2001 From: yanqingmen Date: Tue, 29 Dec 2015 22:42:57 +0800 Subject: [PATCH 14/14] change NDArray writable flag and rm log txt --- .../src/main/scala/ml/dmlc/mxnet/IO.scala | 4 +- scala-package/scala_test_results.txt | 664 ------------------ 2 files changed, 2 insertions(+), 666 deletions(-) delete mode 100644 scala-package/scala_test_results.txt diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala index 758ea2b62f73..11dcfadbbcff 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/IO.scala @@ -157,7 +157,7 @@ class MXDataIter(val handle: DataIterHandle) extends DataIter { override def getData(): NDArray = { val out = new NDArrayHandle checkCall(_LIB.mxDataIterGetData(handle, out)) - return new NDArray(out) + return new NDArray(out, writable = false) } /** @@ -167,7 +167,7 @@ class MXDataIter(val handle: DataIterHandle) extends DataIter { override def getLabel(): NDArray = { val out = new NDArrayHandle checkCall(_LIB.mxDataIterGetLabel(handle, out)) - return new NDArray(out) + return new NDArray(out, writable = false) } /** diff --git a/scala-package/scala_test_results.txt b/scala-package/scala_test_results.txt deleted file mode 100644 index fad7ea59cbae..000000000000 --- a/scala-package/scala_test_results.txt +++ /dev/null @@ -1,664 +0,0 @@ -[INFO] Scanning for projects... -[WARNING] -[WARNING] Some problems were encountered while building the effective model for ml.dmlc.mxnet:mxnet-scala-core:jar:0.1-SNAPSHOT -[WARNING] 'build.plugins.plugin.version' for org.apache.maven.plugins:maven-compiler-plugin is missing. @ line 42, column 15 -[WARNING] -[WARNING] Some problems were encountered while building the effective model for ml.dmlc.mxnet:libmxnet-scala:so:0.0.1-SNAPSHOT -[WARNING] 'build.plugins.plugin.version' for org.apache.maven.plugins:maven-compiler-plugin is missing. @ line 34, column 15 -[WARNING] -[WARNING] It is highly recommended to fix these problems because they threaten the stability of your build. -[WARNING] -[WARNING] For this reason, future Maven versions might no longer support building such malformed projects. -[WARNING] -[INFO] ------------------------------------------------------------------------ -[INFO] Reactor Build Order: -[INFO] -[INFO] MXNet Scala Package - Parent -[INFO] MXNet Scala Package - Core -[INFO] MXNet Scala Package - Native Parent -[INFO] MXNet Scala Package - Native Linux-x86_64 -[INFO] -[INFO] ------------------------------------------------------------------------ -[INFO] Building MXNet Scala Package - Parent 0.1-SNAPSHOT -[INFO] ------------------------------------------------------------------------ -[INFO] -[INFO] ------------------------------------------------------------------------ -[INFO] Building MXNet Scala Package - Core 0.1-SNAPSHOT -[INFO] ------------------------------------------------------------------------ -[INFO] -[INFO] --- maven-resources-plugin:2.7:copy-resources (copy-resources) @ mxnet-scala-core --- -[WARNING] File encoding has not been set, using platform encoding UTF-8, i.e. build is platform dependent! -[WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! -[INFO] Copying 2 resources -[INFO] -[INFO] --- maven-resources-plugin:2.7:resources (default-resources) @ mxnet-scala-core --- -[WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! -[INFO] Copying 2 resources -[INFO] -[INFO] --- scala-maven-plugin:3.2.2:compile (default) @ mxnet-scala-core --- -[INFO] Nothing to compile - all classes are up to date -[INFO] -[INFO] --- maven-compiler-plugin:2.0.2:compile (default-compile) @ mxnet-scala-core --- -[INFO] No sources to compile -[INFO] -[INFO] --- scala-maven-plugin:3.2.2:compile (compile) @ mxnet-scala-core --- -[INFO] Nothing to compile - all classes are up to date -[INFO] -[INFO] --- maven-resources-plugin:2.7:testResources (default-testResources) @ mxnet-scala-core --- -[WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! -[INFO] skip non existing resourceDirectory /home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/core/src/test/resources -[INFO] -[INFO] --- maven-compiler-plugin:2.0.2:testCompile (default-testCompile) @ mxnet-scala-core --- -[INFO] No sources to compile -[INFO] -[INFO] --- scala-maven-plugin:3.2.2:testCompile (test-compile) @ mxnet-scala-core --- -[INFO] Nothing to compile - all classes are up to date -[INFO] -[INFO] --- maven-surefire-plugin:2.7:test (default-test) @ mxnet-scala-core --- -[INFO] Tests are skipped. -[INFO] -[INFO] --- maven-jar-plugin:2.5:jar (default-jar) @ mxnet-scala-core --- -[INFO] Building jar: /home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/core/target/mxnet-scala-core-0.1-SNAPSHOT.jar -[INFO] -[INFO] --- maven-dependency-plugin:2.9:copy-dependencies (copy-dependencies) @ mxnet-scala-core --- -[INFO] log4j:log4j:jar:1.2.17 already exists in destination. -[INFO] org.slf4j:slf4j-api:jar:1.7.5 already exists in destination. -[INFO] org.slf4j:slf4j-log4j12:jar:1.7.7 already exists in destination. -[INFO] org.scala-lang:scala-library:jar:2.10.4 already exists in destination. -[INFO] commons-codec:commons-codec:jar:1.10 already exists in destination. -[INFO] -[INFO] --- maven-jar-plugin:2.5:jar (default) @ mxnet-scala-core --- -[INFO] -[INFO] --- scalatest-maven-plugin:1.0:test (test) @ mxnet-scala-core --- -Discovery starting. -Discovery completed in 183 milliseconds. -Run starting. Expected test count is: 19 -KVStoreSuite: -2015-12-28 23:08:03,245 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -_set_value - - -Parameters ----------- -src : real_t -Source input to the function. - - -2015-12-28 23:08:03,245 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -_plus - - -Parameters ----------- -lhs : NDArray -Left operand to the function. -rhs : NDArray -Right operand to the function. - - -2015-12-28 23:08:03,246 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -_minus - - -Parameters ----------- -lhs : NDArray -Left operand to the function. -rhs : NDArray -Right operand to the function. - - -2015-12-28 23:08:03,246 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -_mul - - -Parameters ----------- -lhs : NDArray -Left operand to the function. -rhs : NDArray -Right operand to the function. - - -2015-12-28 23:08:03,247 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -_div - - -Parameters ----------- -lhs : NDArray -Left operand to the function. -rhs : NDArray -Right operand to the function. - - -2015-12-28 23:08:03,247 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -dot -Calcuate 2D matrix multiplication - -Parameters ----------- -lhs : NDArray -Left operand to the function. -rhs : NDArray -Right operand to the function. - - -2015-12-28 23:08:03,247 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -_onehot_encode - - -Parameters ----------- -lhs : NDArray -Left operand to the function. -rhs : NDArray -Right operand to the function. - - -2015-12-28 23:08:03,248 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -choose_element_0index -Choose one element from each line(row for python, column for R/Julia) in lhs according to index indicated by rhs. This function assume rhs uses 0-based index. - -Parameters ----------- -lhs : NDArray -Left operand to the function. -rhs : NDArray -Right operand to the function. - - -2015-12-28 23:08:03,248 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -_plus_scalar - - -Parameters ----------- -lhs : NDArray -Left operand to the function. -rhs : real_t -Right operand to the function. - - -2015-12-28 23:08:03,249 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -_minus_scalar - - -Parameters ----------- -lhs : NDArray -Left operand to the function. -rhs : real_t -Right operand to the function. - - -2015-12-28 23:08:03,249 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -_mul_scalar - - -Parameters ----------- -lhs : NDArray -Left operand to the function. -rhs : real_t -Right operand to the function. - - -2015-12-28 23:08:03,249 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -_div_scalar - - -Parameters ----------- -lhs : NDArray -Left operand to the function. -rhs : real_t -Right operand to the function. - - -2015-12-28 23:08:03,250 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -_rminus_scalar - - -Parameters ----------- -lhs : NDArray -Left operand to the function. -rhs : real_t -Right operand to the function. - - -2015-12-28 23:08:03,250 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -_rdiv_scalar - - -Parameters ----------- -lhs : NDArray -Left operand to the function. -rhs : real_t -Right operand to the function. - - -2015-12-28 23:08:03,250 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -_copyto - - -Parameters ----------- -src : NDArray -Source input to the function. - - -2015-12-28 23:08:03,251 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -_random_uniform - - -Parameters ----------- - - - -2015-12-28 23:08:03,251 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -_random_gaussian - - -Parameters ----------- - - - -2015-12-28 23:08:03,251 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -clip -Clip ndarray elements to range (a_min, a_max) - -Parameters ----------- -src : NDArray -Source input -a_min : real_t -Minimum value -a_max : real_t -Maximum value - - -2015-12-28 23:08:03,252 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -abs -Take absolute value of the src - -Parameters ----------- -src : NDArray -Source input to the function - - -2015-12-28 23:08:03,252 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -sign -Take sign value of the src - -Parameters ----------- -src : NDArray -Source input to the function - - -2015-12-28 23:08:03,252 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -round -Take round value of the src - -Parameters ----------- -src : NDArray -Source input to the function - - -2015-12-28 23:08:03,253 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -ceil -Take ceil value of the src - -Parameters ----------- -src : NDArray -Source input to the function - - -2015-12-28 23:08:03,253 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -floor -Take floor value of the src - -Parameters ----------- -src : NDArray -Source input to the function - - -2015-12-28 23:08:03,253 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -square -Take square of the src - -Parameters ----------- -src : NDArray -Source input to the function - - -2015-12-28 23:08:03,253 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -sqrt -Take sqrt of the src - -Parameters ----------- -src : NDArray -Source input to the function - - -2015-12-28 23:08:03,254 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -rsqrt -Take rsqrt of the src - -Parameters ----------- -src : NDArray -Source input to the function - - -2015-12-28 23:08:03,254 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -exp -Take exp of the src - -Parameters ----------- -src : NDArray -Source input to the function - - -2015-12-28 23:08:03,254 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -log -Take log of the src - -Parameters ----------- -src : NDArray -Source input to the function - - -2015-12-28 23:08:03,255 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -cos -Take cos of the src - -Parameters ----------- -src : NDArray -Source input to the function - - -2015-12-28 23:08:03,255 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -sin -Take sin of the src - -Parameters ----------- -src : NDArray -Source input to the function - - -2015-12-28 23:08:03,255 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -norm -Take L2 norm of the src.The result will be ndarray of shape (1,) on the same device. - -Parameters ----------- -src : NDArray -Source input to the function - - -2015-12-28 23:08:03,256 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -max -Take max of the src.The result will be ndarray of shape (1,) on the same device. - -Parameters ----------- -src : NDArray -Source input to the function - - -2015-12-28 23:08:03,256 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -min -Take min of the src.The result will be ndarray of shape (1,) on the same device. - -Parameters ----------- -src : NDArray -Source input to the function - - -2015-12-28 23:08:03,256 [ScalaTest-main-running-KVStoreSuite] [ml.dmlc.mxnet.NDArray] [DEBUG] - NDArray function defination: -sum -Take sum of the src.The result will be ndarray of shape (1,) on the same device. - -Parameters ----------- -src : NDArray -Source input to the function - - -- init and pull -- push and pull -update on key 3 -- updater runs when push -- get type -- get numWorkers and rank -2015-12-28 23:08:03,303 [ScalaTest-main-running-IOSuite] [ml.dmlc.mxnet.DataIter] [DEBUG] - ImageRecordIter -Create iterator for dataset packed in recordio. - -Parameters ----------- -path_imglist : string, optional, default='' -Dataset Param: Path to image list. -path_imgrec : string, optional, default='./data/imgrec.rec' -Dataset Param: Path to image record file. -label_width : int, optional, default='1' -Dataset Param: How many labels for an image. -data_shape : Shape(tuple), required -Dataset Param: Shape of each instance generated by the DataIter. -preprocess_threads : int, optional, default='4' -Backend Param: Number of thread to do preprocessing. -verbose : boolean, optional, default=True -Auxiliary Param: Whether to output parser information. -num_parts : int, optional, default='1' -partition the data into multiple parts -part_index : int, optional, default='0' -the index of the part will read -shuffle : boolean, optional, default=False -Augmentation Param: Whether to shuffle data. -seed : int, optional, default='0' -Augmentation Param: Random Seed. -verbose : boolean, optional, default=True -Auxiliary Param: Whether to output information. -batch_size : int (non-negative), required -Batch Param: Batch size. -round_batch : boolean, optional, default=True -Batch Param: Use round robin to handle overflow batch. -prefetch_buffer : long (non-negative), optional, default=4 -Backend Param: Number of prefetched parameters -rand_crop : boolean, optional, default=False -Augmentation Param: Whether to random crop on the image -crop_y_start : int, optional, default='-1' -Augmentation Param: Where to nonrandom crop on y. -crop_x_start : int, optional, default='-1' -Augmentation Param: Where to nonrandom crop on x. -max_rotate_angle : int, optional, default='0' -Augmentation Param: rotated randomly in [-max_rotate_angle, max_rotate_angle]. -max_aspect_ratio : float, optional, default=0 -Augmentation Param: denotes the max ratio of random aspect ratio augmentation. -max_shear_ratio : float, optional, default=0 -Augmentation Param: denotes the max random shearing ratio. -max_crop_size : int, optional, default='-1' -Augmentation Param: Maximum crop size. -min_crop_size : int, optional, default='-1' -Augmentation Param: Minimum crop size. -max_random_scale : float, optional, default=1 -Augmentation Param: Maxmum scale ratio. -min_random_scale : float, optional, default=1 -Augmentation Param: Minimum scale ratio. -max_img_size : float, optional, default=1e+10 -Augmentation Param: Maxmum image size after resizing. -min_img_size : float, optional, default=0 -Augmentation Param: Minimum image size after resizing. -rotate : int, optional, default='-1' -Augmentation Param: Rotate angle. -fill_value : int, optional, default='255' -Augmentation Param: Maximum value of illumination variation. -data_shape : Shape(tuple), required -Dataset Param: Shape of each instance generated by the DataIter. -seed : int, optional, default='0' -Augmentation Param: Random Seed. -mirror : boolean, optional, default=False -Augmentation Param: Whether to mirror the image. -rand_mirror : boolean, optional, default=False -Augmentation Param: Whether to mirror the image randomly. -mean_img : string, optional, default='' -Augmentation Param: Mean Image to be subtracted. -mean_r : float, optional, default=0 -Augmentation Param: Mean value on R channel. -mean_g : float, optional, default=0 -Augmentation Param: Mean value on G channel. -mean_b : float, optional, default=0 -Augmentation Param: Mean value on B channel. -mean_a : float, optional, default=0 -Augmentation Param: Mean value on Alpha channel. -scale : float, optional, default=1 -Augmentation Param: Scale in color space. -max_random_contrast : float, optional, default=0 -Augmentation Param: Maximum ratio of contrast variation. -max_random_illumination : float, optional, default=0 -Augmentation Param: Maximum value of illumination variation. -verbose : boolean, optional, default=True -Augmentation Param: Whether to print augmentor info. - - -2015-12-28 23:08:03,304 [ScalaTest-main-running-IOSuite] [ml.dmlc.mxnet.DataIter] [DEBUG] - MNISTIter -Create iterator for MNIST hand-written digit number recognition dataset. - -Parameters ----------- -image : string, optional, default='./train-images-idx3-ubyte' -Dataset Param: Mnist image path. -label : string, optional, default='./train-labels-idx1-ubyte' -Dataset Param: Mnist label path. -batch_size : int, optional, default='128' -Batch Param: Batch Size. -shuffle : boolean, optional, default=True -Augmentation Param: Whether to shuffle data. -flat : boolean, optional, default=False -Augmentation Param: Whether to flat the data into 1D. -seed : int, optional, default='0' -Augmentation Param: Random Seed. -silent : boolean, optional, default=False -Auxiliary Param: Whether to print out data info. -num_parts : int, optional, default='1' -partition the data into multiple parts -part_index : int, optional, default='0' -the index of the part will read -prefetch_buffer : long (non-negative), optional, default=4 -Backend Param: Number of prefetched parameters - - -2015-12-28 23:08:03,305 [ScalaTest-main-running-IOSuite] [ml.dmlc.mxnet.DataIter] [DEBUG] - CSVIter -Create iterator for dataset in csv. - -Parameters ----------- -data_csv : string, required -Dataset Param: Data csv path. -data_shape : Shape(tuple), required -Dataset Param: Shape of the data. -label_csv : string, optional, default='NULL' -Dataset Param: Label csv path. If is NULL, all labels will be returned as 0 -label_shape : Shape(tuple), optional, default=(1,) -Dataset Param: Shape of the label. - - -IOSuite: -[23:08:04] src/io/iter_mnist.cc:91: MNISTIter: load 60000 images, shuffle=1, shape=(100,784) -- test MNISTIter -NDArraySuite: -- to java array -- to scalar -- call toScalar on an ndarray which is not a scalar -- size and shape -- set scalar value -- copy from java array -- plus -- minus -- multiplication -- division -- clip -- sqrt -- norm -Run completed in 1 second, 908 milliseconds. -Total number of tests run: 19 -Suites: completed 4, aborted 0 -Tests: succeeded 19, failed 0, canceled 0, ignored 0, pending 0 -All tests passed. -[INFO] -[INFO] ------------------------------------------------------------------------ -[INFO] Building MXNet Scala Package - Native Parent 0.1-SNAPSHOT -[INFO] ------------------------------------------------------------------------ -[INFO] -[INFO] ------------------------------------------------------------------------ -[INFO] Building MXNet Scala Package - Native Linux-x86_64 0.0.1-SNAPSHOT -[INFO] ------------------------------------------------------------------------ -[INFO] -[INFO] --- maven-resources-plugin:2.7:copy-resources (copy-resources) @ libmxnet-scala --- -[WARNING] File encoding has not been set, using platform encoding UTF-8, i.e. build is platform dependent! -[WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! -[INFO] skip non existing resourceDirectory /home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/native/linux-x86_64/src/main/resources -[INFO] -[INFO] --- native-maven-plugin:1.0-alpha-7:initialize (default-initialize) @ libmxnet-scala --- -[INFO] -[INFO] --- native-maven-plugin:1.0-alpha-7:unzipinc (default-unzipinc) @ libmxnet-scala --- -[INFO] -[INFO] --- native-maven-plugin:1.0-alpha-7:javah (default-javah) @ libmxnet-scala --- -[INFO] -[INFO] --- maven-antrun-plugin:1.7:run (default) @ libmxnet-scala --- -[WARNING] Parameter tasks is deprecated, use target instead -[INFO] Executing tasks - -main: - [delete] Deleting: /home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/native/linux-x86_64/opencv.pkg.txt -[INFO] Executed tasks -[INFO] -[INFO] --- native-maven-plugin:1.0-alpha-7:javah (javah) @ libmxnet-scala --- -[INFO] /bin/sh -c cd /home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/native/linux-x86_64 && javah -o /home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/native/linux-x86_64/target/custom-javah/ml_dmlc_mxnet_native_c_api.h -classpath /home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/native/linux-x86_64/target/classes:/home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/core/target/mxnet-scala-core-0.1-SNAPSHOT.jar:/home/hzx/.m2/repository/org/scala-lang/scala-library/2.10.4/scala-library-2.10.4.jar:/home/hzx/.m2/repository/commons-codec/commons-codec/1.10/commons-codec-1.10.jar:/home/hzx/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/home/hzx/.m2/repository/org/slf4j/slf4j-api/1.7.5/slf4j-api-1.7.5.jar:/home/hzx/.m2/repository/org/slf4j/slf4j-log4j12/1.7.7/slf4j-log4j12-1.7.7.jar ml.dmlc.mxnet.LibInfo -[INFO] -[INFO] --- native-maven-plugin:1.0-alpha-7:compile (default-compile) @ libmxnet-scala --- -[INFO] -[INFO] --- native-maven-plugin:1.0-alpha-7:link (default-link) @ libmxnet-scala --- -[INFO] /bin/sh -c cd /home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/native/linux-x86_64 && g++-4.8 -shared -o/home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/native/linux-x86_64/target/libmxnet-scala.so target/objs/ml_dmlc_mxnet_native_c_api.o -Wl,--whole-archive ../../../lib/libmxnet.a ../../../dmlc-core/libdmlc.a -Wl,--no-whole-archive -lm -lrt -fopenmp /usr/lib/x86_64-linux-gnu/libopencv_calib3d.so /usr/lib/x86_64-linux-gnu/libopencv_contrib.so /usr/lib/x86_64-linux-gnu/libopencv_core.so /usr/lib/x86_64-linux-gnu/libopencv_features2d.so /usr/lib/x86_64-linux-gnu/libopencv_flann.so /usr/lib/x86_64-linux-gnu/libopencv_gpu.so /usr/lib/x86_64-linux-gnu/libopencv_highgui.so /usr/lib/x86_64-linux-gnu/libopencv_imgproc.so /usr/lib/x86_64-linux-gnu/libopencv_legacy.so /usr/lib/x86_64-linux-gnu/libopencv_ml.so /usr/lib/x86_64-linux-gnu/libopencv_objdetect.so /usr/lib/x86_64-linux-gnu/libopencv_ocl.so /usr/lib/x86_64-linux-gnu/libopencv_photo.so /usr/lib/x86_64-linux-gnu/libopencv_stitching.so /usr/lib/x86_64-linux-gnu/libopencv_superres.so /usr/lib/x86_64-linux-gnu/libopencv_ts.so /usr/lib/x86_64-linux-gnu/libopencv_video.so /usr/lib/x86_64-linux-gnu/libopencv_videostab.so -lopencv_calib3d -lopencv_contrib -lopencv_core -lopencv_features2d -lopencv_flann -lopencv_gpu -lopencv_highgui -lopencv_imgproc -lopencv_legacy -lopencv_ml -lopencv_objdetect -lopencv_ocl -lopencv_photo -lopencv_stitching -lopencv_superres -lopencv_ts -lopencv_video -lopencv_videostab -lblas -[INFO] -[INFO] --- maven-resources-plugin:2.7:testResources (default-testResources) @ libmxnet-scala --- -[WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! -[INFO] skip non existing resourceDirectory /home/hzx/workspace/git/mxnet-scala/mxnet/scala-package/native/linux-x86_64/src/test/resources -[INFO] -[INFO] --- maven-compiler-plugin:3.3:testCompile (default-testCompile) @ libmxnet-scala --- -[INFO] No sources to compile -[INFO] -[INFO] --- maven-surefire-plugin:2.7:test (default-test) @ libmxnet-scala --- -[INFO] Tests are skipped. -[INFO] -[INFO] --- native-maven-plugin:1.0-alpha-7:inczip (default-inczip) @ libmxnet-scala --- -[INFO] ------------------------------------------------------------------------ -[INFO] Reactor Summary: -[INFO] -[INFO] MXNet Scala Package - Parent ...................... SUCCESS [0.033s] -[INFO] MXNet Scala Package - Core ........................ SUCCESS [4.727s] -[INFO] MXNet Scala Package - Native Parent ............... SUCCESS [0.001s] -[INFO] MXNet Scala Package - Native Linux-x86_64 ......... SUCCESS [2.198s] -[INFO] ------------------------------------------------------------------------ -[INFO] BUILD SUCCESS -[INFO] ------------------------------------------------------------------------ -[INFO] Total time: 7.507s -[INFO] Finished at: Mon Dec 28 23:08:07 CST 2015 -[INFO] Final Memory: 24M/341M -[INFO] ------------------------------------------------------------------------