From a451a96861900ac435182ef2c7ea4a993b783bc0 Mon Sep 17 00:00:00 2001 From: Yuxi Hu Date: Wed, 23 Jan 2019 21:53:01 -0800 Subject: [PATCH 1/5] create NDArray with CPUPinned context in ImageRecordIOParser2 --- src/io/image_iter_common.h | 10 +++++++--- src/io/iter_image_recordio_2.cc | 5 +++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/io/image_iter_common.h b/src/io/image_iter_common.h index c9e3933ade28..b24e3bc0b529 100644 --- a/src/io/image_iter_common.h +++ b/src/io/image_iter_common.h @@ -125,11 +125,13 @@ struct ImageRecParserParam : public dmlc::Parameter { bool verbose; /*! \brief partition the data into multiple parts */ int num_parts; - /*! \brief the index of the part will read*/ + /*! \brief the index of the part will read */ int part_index; - /*! \brief the size of a shuffle chunk*/ + /*! \brief device id on which to store the data */ + int device_id; + /*! \brief the size of a shuffle chunk */ size_t shuffle_chunk_size; - /*! \brief the seed for chunk shuffling*/ + /*! \brief the seed for chunk shuffling */ int shuffle_chunk_seed; /*! \brief random seed for augmentations */ dmlc::optional seed_aug; @@ -163,6 +165,8 @@ struct ImageRecParserParam : public dmlc::Parameter { .describe("Virtually partition the data into these many parts."); DMLC_DECLARE_FIELD(part_index).set_default(0) .describe("The *i*-th virtual partition to be read."); + DMLC_DECLARE_FIELD(device_id).set_default(0) + .describe("The device id on which to store the data."); DMLC_DECLARE_FIELD(shuffle_chunk_size).set_default(0) .describe("The data shuffle buffer size in MB. Only valid if shuffle is true."); DMLC_DECLARE_FIELD(shuffle_chunk_seed).set_default(0) diff --git a/src/io/iter_image_recordio_2.cc b/src/io/iter_image_recordio_2.cc index 89f7753983db..95ed83c1d82a 100644 --- a/src/io/iter_image_recordio_2.cc +++ b/src/io/iter_image_recordio_2.cc @@ -285,9 +285,10 @@ inline bool ImageRecordIOParser2::ParseNext(DataBatch *out) { shape_vec.push_back(param_.label_width); TShape label_shape(shape_vec.begin(), shape_vec.end()); - out->data.at(0) = NDArray(data_shape, Context::CPU(0), false, + auto dev_id = param_.device_id; + out->data.at(0) = NDArray(data_shape, Context::CPUPinned(dev_id), false, mshadow::DataType::kFlag); - out->data.at(1) = NDArray(label_shape, Context::CPU(0), false, + out->data.at(1) = NDArray(label_shape, Context::CPUPinned(dev_id), false, mshadow::DataType::kFlag); unit_size_[0] = param_.data_shape.Size(); unit_size_[1] = param_.label_width; From 70b1b2c0046d2d07c7aad51169d86f9f106d556e Mon Sep 17 00:00:00 2001 From: Yuxi Hu Date: Thu, 24 Jan 2019 14:33:24 -0800 Subject: [PATCH 2/5] update document --- src/io/image_iter_common.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/io/image_iter_common.h b/src/io/image_iter_common.h index b24e3bc0b529..99e6658d3d27 100644 --- a/src/io/image_iter_common.h +++ b/src/io/image_iter_common.h @@ -127,7 +127,7 @@ struct ImageRecParserParam : public dmlc::Parameter { int num_parts; /*! \brief the index of the part will read */ int part_index; - /*! \brief device id on which to store the data */ + /*! \brief device id used to create CPUPinned context for internal NDArray */ int device_id; /*! \brief the size of a shuffle chunk */ size_t shuffle_chunk_size; @@ -166,7 +166,7 @@ struct ImageRecParserParam : public dmlc::Parameter { DMLC_DECLARE_FIELD(part_index).set_default(0) .describe("The *i*-th virtual partition to be read."); DMLC_DECLARE_FIELD(device_id).set_default(0) - .describe("The device id on which to store the data."); + .describe("The device id used to create CPUPinned context for internal NDArray."); DMLC_DECLARE_FIELD(shuffle_chunk_size).set_default(0) .describe("The data shuffle buffer size in MB. Only valid if shuffle is true."); DMLC_DECLARE_FIELD(shuffle_chunk_seed).set_default(0) From 3776357a8701ec69358f3f0d887e8192c297f316 Mon Sep 17 00:00:00 2001 From: Yuxi Hu Date: Thu, 24 Jan 2019 18:07:11 -0800 Subject: [PATCH 3/5] use -1 device_id as an option to create CPU(0) context --- src/io/image_iter_common.h | 6 ++++-- src/io/iter_image_recordio_2.cc | 8 ++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/io/image_iter_common.h b/src/io/image_iter_common.h index 99e6658d3d27..4e01f4598894 100644 --- a/src/io/image_iter_common.h +++ b/src/io/image_iter_common.h @@ -127,7 +127,7 @@ struct ImageRecParserParam : public dmlc::Parameter { int num_parts; /*! \brief the index of the part will read */ int part_index; - /*! \brief device id used to create CPUPinned context for internal NDArray */ + /*! \brief device id used to create context for internal NDArray */ int device_id; /*! \brief the size of a shuffle chunk */ size_t shuffle_chunk_size; @@ -166,7 +166,9 @@ struct ImageRecParserParam : public dmlc::Parameter { DMLC_DECLARE_FIELD(part_index).set_default(0) .describe("The *i*-th virtual partition to be read."); DMLC_DECLARE_FIELD(device_id).set_default(0) - .describe("The device id used to create CPUPinned context for internal NDArray."); + .describe("The device id used to create context for internal NDArray. "\ + "-1 will create Context::CPU(0). Valid positive device "\ + "id will create Context::CPUPinned(device_id). Default is 0." ); DMLC_DECLARE_FIELD(shuffle_chunk_size).set_default(0) .describe("The data shuffle buffer size in MB. Only valid if shuffle is true."); DMLC_DECLARE_FIELD(shuffle_chunk_seed).set_default(0) diff --git a/src/io/iter_image_recordio_2.cc b/src/io/iter_image_recordio_2.cc index 95ed83c1d82a..00c38198659f 100644 --- a/src/io/iter_image_recordio_2.cc +++ b/src/io/iter_image_recordio_2.cc @@ -285,10 +285,14 @@ inline bool ImageRecordIOParser2::ParseNext(DataBatch *out) { shape_vec.push_back(param_.label_width); TShape label_shape(shape_vec.begin(), shape_vec.end()); + auto ctx = Context::CPU(0); auto dev_id = param_.device_id; - out->data.at(0) = NDArray(data_shape, Context::CPUPinned(dev_id), false, + if (dev_id != -1) { + ctx = Context::CPUPinned(dev_id); + } + out->data.at(0) = NDArray(data_shape, ctx, false, mshadow::DataType::kFlag); - out->data.at(1) = NDArray(label_shape, Context::CPUPinned(dev_id), false, + out->data.at(1) = NDArray(label_shape, ctx, false, mshadow::DataType::kFlag); unit_size_[0] = param_.data_shape.Size(); unit_size_[1] = param_.label_width; From 1b93fd4c635a5ad41dec8eaa1b40475d34ccd9f6 Mon Sep 17 00:00:00 2001 From: Yuxi Hu Date: Thu, 24 Jan 2019 18:51:46 -0800 Subject: [PATCH 4/5] retrigger CI From 156fcc8d3cafafc9e768e1418f01abc5d5ec7365 Mon Sep 17 00:00:00 2001 From: Yuxi Hu Date: Thu, 24 Jan 2019 19:02:20 -0800 Subject: [PATCH 5/5] fix cpplint error --- src/io/image_iter_common.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/io/image_iter_common.h b/src/io/image_iter_common.h index 4e01f4598894..10cd8ab4e5de 100644 --- a/src/io/image_iter_common.h +++ b/src/io/image_iter_common.h @@ -167,8 +167,9 @@ struct ImageRecParserParam : public dmlc::Parameter { .describe("The *i*-th virtual partition to be read."); DMLC_DECLARE_FIELD(device_id).set_default(0) .describe("The device id used to create context for internal NDArray. "\ - "-1 will create Context::CPU(0). Valid positive device "\ - "id will create Context::CPUPinned(device_id). Default is 0." ); + "Setting device_id to -1 will create Context::CPU(0). Setting " + "device_id to valid positive device id will create " + "Context::CPUPinned(device_id). Default is 0."); DMLC_DECLARE_FIELD(shuffle_chunk_size).set_default(0) .describe("The data shuffle buffer size in MB. Only valid if shuffle is true."); DMLC_DECLARE_FIELD(shuffle_chunk_seed).set_default(0)