Local IO

Makefile

@@ -57,7 +57,7 @@ endif
 #BIN = test/test_threaded_engine test/api_registry_test
 OBJ = narray_function_cpu.o
 # add threaded engine after it is done
-OBJCXX11 = engine.o narray.o c_api.o operator.o symbol.o storage.o fully_connected_cpu.o static_graph.o activation_cpu.o graph_executor.o softmax_cpu.o elementwise_sum_cpu.o pooling_cpu.o
+OBJCXX11 = engine.o narray.o c_api.o operator.o symbol.o storage.o fully_connected_cpu.o static_graph.o activation_cpu.o graph_executor.o softmax_cpu.o elementwise_sum_cpu.o pooling_cpu.o io.o iter_mnist.o
 CUOBJ =
 SLIB = lib/libmxnet.so
 ALIB = lib/libmxnet.a
@@ -94,11 +94,14 @@ pooling_cpu.o: src/operator/pooling.cc
 pooling_gpu.o: src/operator/pooling.cu
 softmax_cpu.o: src/operator/softmax.cc
 softmax_gpu.o: src/operator/softmax.cu
+io.o: src/io/io.cc
+iter_mnist.o: src/io/iter_mnist.cc
 
-lib/libmxnet.a: $(OBJ) $(OBJCXX11) $(CUOBJ)
-lib/libmxnet.so: $(OBJ) $(OBJCXX11) $(CUOBJ)
+lib/libmxnet.a: $(OBJ) $(OBJCXX11) $(CUOBJ) $(LIB_DEP)
+lib/libmxnet.so: $(OBJ) $(OBJCXX11) $(CUOBJ) $(LIB_DEP)
 
 test/test_storage: test/test_storage.cc lib/libmxnet.a
+#test/io_mnist_test: test/io_mnist_test.cc lib/libmxnet.a $(DMLC_CORE)/libdmlc.a
 #test/test_threaded_engine: test/test_threaded_engine.cc api/libmxnet.a
 
 $(BIN) :

include/mxnet/c_api.h

@@ -36,6 +36,8 @@ typedef void *SymbolHandle;
 typedef void *AtomicSymbolHandle;
 /*! \brief handle to an Executor */
 typedef void *ExecutorHandle;
+/*! \brief handle a dataiter creator */
+typedef void *DataIterCreator;
 /*! \brief handle to a DataIterator */
 typedef void *DataIterHandle;
 /*!
@@ -439,49 +441,90 @@ MXNET_DLL int MXExecutorBind(SymbolHandle symbol_handle,
 // Part 5: IO Interface
 //--------------------------------------------
 /*!
- * \brief create an data iterator from configs string
- * \param cfg config string that contains the
- *    configuration about the iterator
- * \param out the handle to the iterator
+ * \brief list all the available iterator entries
+ * \param out_size the size of returned iterators
+ * \param out_array the output iteratos entries
  * \return 0 when success, -1 when failure happens
  */
-MXNET_DLL int MXIOCreateFromConfig(const char *cfg,
-                                   DataIterHandle *out);
+MXNET_DLL int MXListDataIters(mx_uint *out_size,
+                              DataIterCreator **out_array);
+/*!
+ * \brief get the name of iterator entry
+ * \param iter iterator entry
+ * \param out_name the name of the iterator
+ * \return 0 when success, -1 when failure happens
+ */
+MXNET_DLL int MXDataIterGetName(DataIterCreator iter,
+                            const char **out_name);
+/*!
+ * \brief init an iterator, init with parameters
+ * the array size of passed in arguments
+ * \param handle of the iterator creator
+ * \param num_param number of parameter
+ * \param keys parameter keys
+ * \param vals parameter values
+ * \param out resulting iterator
+ * \return 0 when success, -1 when failure happens
+ */
+MXNET_DLL int MXDataIterCreateIter(DataIterCreator handle,
+                               int num_param,
+                               const char **keys,
+                               const char **vals,
+                               DataIterHandle *out);
+/*!
+ * \brief Get the detailed information about data iterator.
+ * \param creator the DataIterCreator.
+ * \param name The returned name of the creator.
+ * \param description The returned description of the symbol.
+ * \param num_args Number of arguments.
+ * \param arg_names Name of the arguments.
+ * \param arg_type_infos Type informations about the arguments.
+ * \param arg_descriptions Description information about the arguments.
+ * \return 0 when success, -1 when failure happens
+ */
+MXNET_DLL int MXDataIterGetIterInfo(AtomicSymbolCreator creator,
+                                          const char **name,
+                                          const char **description,
+                                          mx_uint *num_args,
+                                          const char ***arg_names,
+                                          const char ***arg_type_infos,
+                                          const char ***arg_descriptions);
+/*!
+ * \brief free the handle to the IO module
+ * \param handle the handle pointer to the data iterator
+ * \return 0 when success, -1 when failure happens
+ */
+MXNET_DLL int MXDataIterFree(DataIterHandle handle);
 /*!
  * \brief move iterator to next position
  * \param handle the handle to iterator
  * \param out return value of next
  * \return 0 when success, -1 when failure happens
  */
-MXNET_DLL int MXIONext(DataIterHandle handle,
+MXNET_DLL int MXDataIterNext(DataIterHandle handle,
                        int *out);
 /*!
  * \brief call iterator.BeforeFirst
  * \param handle the handle to iterator
  * \return 0 when success, -1 when failure happens
  */
-MXNET_DLL int MXIOBeforeFirst(DataIterHandle handle);
-/*!
- * \brief free the handle to the IO module
- * \param handle the handle pointer to the data iterator
- * \return 0 when success, -1 when failure happens
- */
-MXNET_DLL int MXIOFree(DataIterHandle handle);
+MXNET_DLL int MXDataIterBeforeFirst(DataIterHandle handle);
+
 /*!
  * \brief get the handle to the NArray of underlying data
  * \param handle the handle pointer to the data iterator
  * \param out handle to underlying data NArray
  * \return 0 when success, -1 when failure happens
  */
-MXNET_DLL int MXIOGetData(DataIterHandle handle,
+MXNET_DLL int MXDataIterGetData(DataIterHandle handle,
                           NArrayHandle *out);
 /*!
  * \brief get the handle to the NArray of underlying label
  * \param handle the handle pointer to the data iterator
  * \param out the handle to underlying label NArray
  * \return 0 when success, -1 when failure happens
  */
-MXNET_DLL int MXIOGetLabel(DataIterHandle handle,
+MXNET_DLL int MXDataIterGetLabel(DataIterHandle handle,
                            NArrayHandle *out);
 
 #endif  // MXNET_C_API_H_

include/mxnet/io.h

@@ -0,0 +1,115 @@
+/*!
+ *  Copyright (c) 2015 by Contributors
+ * \file io.h
+ * \brief mxnet io data structure and data iterator
+ */
+#ifndef MXNET_IO_H_
+#define MXNET_IO_H_
+#include <dmlc/data.h>
+#include <dmlc/registry.h>
+#include <vector>
+#include <string>
+#include <utility>
+#include "./base.h"
+
+namespace mxnet {
+/*!
+ * \brief iterator type
+ * \tparam DType data type
+ */
+template<typename DType>
+class IIterator : public dmlc::DataIter<DType> {
+ public:
+  /*!
+   * \brief set the parameters and init iter
+   * \param kwargs key-value pairs
+   */ 
+  virtual void SetInit(const std::vector<std::pair<std::string, std::string> >& kwargs) = 0;
+  /*! \brief initalize the iterator so that we can use the iterator */
+  virtual void Init(void) = 0;
+  /*! \brief set before first of the item */
+  virtual void BeforeFirst(void) = 0;
+  /*! \brief move to next item */
+  virtual bool Next(void) = 0;
+  /*! \brief get current data */
+  virtual const DType &Value(void) const = 0;
+  /*! \brief constructor */
+  virtual ~IIterator(void) {}
+  /*! \brief store the name of each data, it could be used for making NArrays */
+  std::vector<std::string> data_names;
+  /*! \brief set data name to each attribute of data */
+  inline void SetDataName(const std::string data_name){
+    data_names.push_back(data_name);
+  }
+};  // class IIterator
+
+/*! \brief a single data instance */
+struct DataInst {
+  /*! \brief unique id for instance */
+  unsigned index;
+  /*! \brief content of data */
+  std::vector<TBlob> data;
+  /*! \brief extra data to be fed to the network */
+  std::string extra_data;
+};  // struct DataInst
+
+/*!
+ * \brief a standard batch of data commonly used by iterator
+ *      a databatch contains multiple TBlobs. Each Tblobs has
+ *      a name stored in a map. There's no different between
+ *      data and label, how we use them is to see the DNN implementation.
+ */
+struct DataBatch {
+ public:
+  /*! \brief unique id for instance, can be NULL, sometimes is useful */
+  unsigned *inst_index;
+  /*! \brief number of instance */
+  mshadow::index_t batch_size;
+  /*! \brief number of padding elements in this batch,
+       this is used to indicate the last elements in the batch are only padded up to match the batch, and should be discarded */
+  mshadow::index_t num_batch_padd;
+ public:
+  /*! \brief content of dense data, if this DataBatch is dense */
+  std::vector<TBlob> data;
+  /*! \brief extra data to be fed to the network */
+  std::string extra_data;
+ public:
+  /*! \brief constructor */
+  DataBatch(void) {
+    inst_index = NULL;
+    batch_size = 0; num_batch_padd = 0;
+  }
+  /*! \brief giving name to the data */
+  void Naming(std::vector<std::string> names);
+};  // struct DataBatch
+
+/*! \brief typedef the factory function of data iterator */
+typedef IIterator<DataBatch> *(*DataIteratorFactory)();
+/*!
+ * \brief Registry entry for DataIterator factory functions.
+ */
+struct DataIteratorReg
+    : public dmlc::FunctionRegEntryBase<DataIteratorReg,
+                                        DataIteratorFactory> {
+};
+//--------------------------------------------------------------
+// The following part are API Registration of Iterators
+//--------------------------------------------------------------
+/*!
+ * \brief Macro to register Iterators
+ *
+ * \code
+ * // example of registering a mnist iterator
+ * REGISTER_IO_ITERATOR(MNIST, MNISTIterator)
+ * .describe("Mnist data iterator");
+ *
+ * \endcode
+ */
+#define MXNET_REGISTER_IO_ITER(name, DataIteratorType)          \
+  static ::mxnet::IIterator<DataBatch>* __create__ ## DataIteratorType ## __() { \
+    return new DataIteratorType;                                    \
+  }                                                                     \
+  DMLC_REGISTRY_REGISTER(::mxnet::DataIteratorReg, DataIteratorReg, name) \
+  .set_body(__create__ ## DataIteratorType ## __)
+}  // namespace mxnet
+#endif  // MXNET_IO_H_

python/mxnet/__init__.py

@@ -12,6 +12,7 @@
 from .context import Context, current_context
 from . import narray
 from . import symbol
+from . import io
 
 __version__ = "0.1.0"
 

python/mxnet/base.py

@@ -70,7 +70,8 @@ def _load_lib():
 SymbolCreatorHandle = ctypes.c_void_p
 SymbolHandle = ctypes.c_void_p
 ExecutorHandle = ctypes.c_void_p
-
+DataIterCreatorHandle = ctypes.c_void_p
+DataIterHandle = ctypes.c_void_p
 #----------------------------
 # helper function definition
 #----------------------------