From cdfacfcab84b6f8ccd1d9d93fa6eeba82fad34d5 Mon Sep 17 00:00:00 2001 From: Da Zheng Date: Fri, 24 Aug 2018 14:28:49 -0700 Subject: [PATCH 01/11] the first version. --- .../control_flow/ControlFlowTutorial.md | 385 ++++++++++++++++++ 1 file changed, 385 insertions(+) create mode 100644 docs/tutorials/control_flow/ControlFlowTutorial.md diff --git a/docs/tutorials/control_flow/ControlFlowTutorial.md b/docs/tutorials/control_flow/ControlFlowTutorial.md new file mode 100644 index 000000000000..047f34e67647 --- /dev/null +++ b/docs/tutorials/control_flow/ControlFlowTutorial.md @@ -0,0 +1,385 @@ + +MXNet currently provides three control flow operators: `cond`, `foreach` and `while_loop`. Like other MXNet operators, they all have a version for NDArray and a version for Symbol. These two versions have exactly the same semantics. We can take advantage of this and use Gluon hybrid blocks and switch between hybrid and non-hybrid modes seamleessly. + +In this tutorial, we use a few examples to demonstrate the use of control flow operators in Gluon and show how a model that requires control flow is hybridized. + +# Prepare running the code + + +```python +import mxnet as mx +from mxnet.gluon import HybridBlock +``` + +# foreach +`foreach` is defined with the following signature: + +```python +foreach(body, data, init_states, name) => (outputs, states) +``` + +It iterates over the first dimension of the input data (it can be an array or multiple arrays) and run the Python function defined in `body` for every slice from the input arrays. The signature of the `body` function is defined as follows: + +```python +body(data, states) => (outputs, states) +``` + +The inputs of the `body` function have two parts: `data` is a slice of an array (if there is only one input array in `foreach`) or a list of slices (if there are multiple input arrays); `states` are the arrays from the previous iteration. The outputs of the `body` function also have two parts: `outputs` is an array or a list of arrays; `states` is the computation states of the current iteration. `outputs` from all iterations are concatenated as the outputs of `foreach`. + +The pseudocode below illustrates the execution of `foreach`. + +```python +def foreach(body, data, init_states): + states = init_states + outs = [] + + for i in range(data.shape[0]): + s = data[i] + out, states = body(s, states) + outs.append(out) + outs = mx.nd.stack(*outs) + return outs, states +``` + +### Example 1: foreach works like map +`foreach` can work like a map function in a functional language. In this case, the states of foreach can be an empty list, which means the computation doesn't carry computation states across iterations. + +In this example, we use `foreach` to add each element in an array by one. + + +```python +data = mx.nd.arange(5) +print(data) +``` + + + [ 0. 1. 2. 3. 4.] + + + + +```python +def add1(data, _): + return data + 1, [] + +class Map(HybridBlock): + def hybrid_forward(self, F, data): + out, _ = F.contrib.foreach(add1, data, []) + return out + +map_layer = Map() +out = map_layer(data) +print(out) +``` + + + [[ 1.] + [ 2.] + [ 3.] + [ 4.] + [ 5.]] + + + +We can hybridize the block and run the computation again. It should generate the same result. + + +```python +map_layer.hybridize() +out = map_layer(data) +print(out) +``` + + + [[ 1.] + [ 2.] + [ 3.] + [ 4.] + [ 5.]] + + + +### Example 2: foreach works like scan +`foreach` can work like a scan function in a functional language. In this case, the outputs of the Python function is an empty list. + + +```python +def sum(data, state): + return [], state + data + +class Scan(HybridBlock): + def hybrid_forward(self, F, data): + _, state = F.contrib.foreach(sum, data, F.zeros((1))) + return state +scan_layer = Scan() +state = scan_layer(data) +print(data) +print(state) +``` + + + [ 0. 1. 2. 3. 4.] + + + [ 10.] + + + + +```python +scan_layer.hybridize() +state = scan_layer(data) +print(state) +``` + + + [ 10.] + + + +### Example 3: foreach with both outputs and states +This is probably the most common use case of `foreach`. We extend the scan example above and return both output and states. + + +```python +def sum(data, state): + return state + data, state + data + +class ScanV2(HybridBlock): + def hybrid_forward(self, F, data): + out, state = F.contrib.foreach(sum, data, F.zeros((1))) + return out, state +scan_layer = ScanV2() +out, state = scan_layer(data) +print(out) +print(state) +``` + + + [[ 0.] + [ 1.] + [ 3.] + [ 6.] + [ 10.]] + + + [ 10.] + + + + +```python +scan_layer.hybridize() +out, state = scan_layer(data) +print(out) +print(state) +``` + + + [[ 0.] + [ 1.] + [ 3.] + [ 6.] + [ 10.]] + + + [ 10.] + + + +### Example 4: use foreach to run RNN on a variable-length sequence +Previous examples illustrate `foreach` with simple use cases. Here I show an example of processing variable-length sequences with `foreach`. The same idea is used by dynamic_rnn in TensorFlow for processing variable-length sequences. + + +```python +class DynamicRNNLayer(HybridBlock): + def __init__(self, cell, prefix=None, params=None): + super(DynamicRNNLayer, self).__init__(prefix=prefix, params=params) + self.cell = cell + def hybrid_forward(self, F, inputs, begin_state, valid_length): + states = begin_state + zeros = [] + for s in states: + zeros.append(F.zeros_like(s)) + # the last state is the iteration number. + states.append(F.zeros((1))) + def loop_body(inputs, states): + cell_states = states[:-1] + # Get the iteration number from the states. + iter_no = states[-1] + out, new_states = self.cell(inputs, cell_states) + # Copy the old state if we have reached the end of a sequence. + for i, state in enumerate(cell_states): + new_states[i] = F.where(F.broadcast_greater(valid_length, iter_no), + new_states[i], state) + new_states.append(iter_no + 1) + return out, new_states + + outputs, states = F.contrib.foreach(loop_body, inputs, states) + outputs = F.SequenceMask(outputs, sequence_length=valid_length, + use_sequence_length=True, axis=0) + # the last state is the iteration number. We don't need it. + return outputs, states[:-1] +``` + + +```python +seq_len = 10 +batch_size = 2 +input_size = 5 +hidden_size = 6 + +rnn_data = mx.nd.normal(loc=0, scale=1, shape=(seq_len, batch_size, input_size)) +init_states = [mx.nd.normal(loc=0, scale=1, shape=(batch_size, hidden_size)) for i in range(2)] +valid_length = mx.nd.round(mx.nd.random.uniform(low=1, high=10, shape=(batch_size))) + +lstm = DynamicRNNLayer(mx.gluon.rnn.LSTMCell(hidden_size)) +lstm.initialize() +res, states = lstm(rnn_data, [x for x in init_states], valid_length) + +lstm.hybridize() +res, states = lstm(rnn_data, [x for x in init_states], valid_length) +``` + +# while_loop +`while_loop` is defined with the following signature: + +```python +while_loop(cond, body, loop_vars, max_iterations, name) => (outputs, states) +``` +Instead of running over the first dimension of an array, `while_loop` checks a condition function in every iteration and runs a `body` function for computation. The signature of the `body` function is defined as follows: + +```python +body(state1, state2, ...) => (outputs, states) +``` + +The inputs of the `body` function in `while_loop` are a little different from the one in `foreach`. It has a variable number of input arguments. Each input argument is a loop variable and the number of arguments is determined by the number of loop variables. The outputs of the `body` function also have two parts: `outputs` is an array or a list of arrays; `states` is the computation states of the current iteration. Like `foreach`, both `outputs` and `states` can be an empty list. `outputs` from all iterations are concatenated as the outputs of `while_loop`. + +### Example 5: scan with while_loop +`while_loop` is more general than `foreach`. We can also use it to iterate over an array and sum all of its values together. In this example, instead of summing over the entire array, we only sum over the first 4 elements. + +**Note**: the output arrays of the current implementation of `while_loop` is determined by `max_iterations`. As such, even though the while loop in this example runs 4 iterations, it still outputs an array of 5 elements. The last element in the output array is actually filled with a random number. + + +```python +class ScanV2(HybridBlock): + def hybrid_forward(self, F, data): + def sum(state, i): + s = state + data[i] + return s, [s, i + 1] + + def sum_cond(state, i): + return i < 4 + + out, state = F.contrib.while_loop(sum_cond, sum, + [F.zeros((1)), F.zeros((1))], max_iterations=5) + return out, state +scan_layer = ScanV2() +out, state = scan_layer(data) +print(out) +print(state) +``` + + + [[ 0.] + [ 1.] + [ 3.] + [ 6.] + [ 0.]] + + [ + [ 6.] + , + [ 4.] + ] + + +# cond +`cond` is defined with the following signature: + +```python +cond(pred, then_func, else_func, name) +``` + +`cond` checks `pred`, which is a symbol or an NDArray with one element. If its value is true, it calls `then_func`. Otherwise, it calls `else_func`. The signature of `then_func` and `else_func` are as follows: + +```python +func() => [outputs] +``` + +`cond` requires all outputs from `then_func` and `else_func` have the same number of Symbols/NDArrays with the same shapes and data types. + +### Example 6: skip RNN computation with cond +Example 4 shows how to process a batch with sequences of different lengths. In this example, we show how to skip computation after we have met the end of the sequence. + + +```python +class SkipRNNCell(HybridBlock): + def __init__(self, cell, prefix=None, params=None): + super(SkipRNNCell, self).__init__(prefix=prefix, params=params) + self.cell = cell + def hybrid_forward(self, F, i, length, data, states): + def run_rnn(): + return self.cell(data, states) + + def copy_states(): + return F.zeros_like(data), states + out, state = F.contrib.cond(i < length, run_rnn, copy_states) + return out, state + +class RNNLayer(HybridBlock): + def __init__(self, cell, prefix=None, params=None): + super(RNNLayer, self).__init__(prefix=prefix, params=params) + self.cell = SkipRNNCell(cell) + def hybrid_forward(self, F, length, data, init_states): + def body(data, states): + i = states[0] + out, states = self.cell(i, length, data, states[1]) + return out, [i + 1, states] + + out, state = F.contrib.foreach(body, data, [F.zeros((1)), init_states]) + return out, state +``` + + +```python +seq_len = 5 +batch_size = 1 +input_size = 3 +hidden_size = 3 + +rnn_data = mx.nd.normal(loc=0, scale=1, shape=(seq_len, batch_size, input_size)) +init_states = [mx.nd.normal(loc=0, scale=1, shape=(batch_size, hidden_size)) for i in range(2)] + +cell = mx.gluon.rnn.LSTMCell(hidden_size) +layer = RNNLayer(cell) +layer.initialize() + +out, states = layer(mx.nd.array([3]), rnn_data, init_states) +print(rnn_data) +print(out) +``` + + + [[[-0.70893967 0.79505837 1.01872194]] + + [[-0.15738758 -1.5379014 -0.78309226]] + + [[ 0.5364728 -1.49940979 0.18817241]] + + [[-0.52421683 0.29636207 -0.53233677]] + + [[ 0.85569632 -0.87393355 -0.10898105]]] + + + [[[ 0.32939154 -0.01365058 -0.25545752]] + + [[ 0.15680683 -0.00335424 -0.14627317]] + + [[ 0.07623718 -0.00953147 -0.09720358]] + + [[ 0. 0. 0. ]] + + [[ 0. 0. 0. ]]] + + From a0e08db55f7a33a10c6ecd3e37fe3bd1c94dff0c Mon Sep 17 00:00:00 2001 From: Da Zheng Date: Fri, 24 Aug 2018 15:42:02 -0700 Subject: [PATCH 02/11] fix. --- .../control_flow/ControlFlowTutorial.md | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/docs/tutorials/control_flow/ControlFlowTutorial.md b/docs/tutorials/control_flow/ControlFlowTutorial.md index 047f34e67647..ddc1f5f8d709 100644 --- a/docs/tutorials/control_flow/ControlFlowTutorial.md +++ b/docs/tutorials/control_flow/ControlFlowTutorial.md @@ -1,5 +1,5 @@ -MXNet currently provides three control flow operators: `cond`, `foreach` and `while_loop`. Like other MXNet operators, they all have a version for NDArray and a version for Symbol. These two versions have exactly the same semantics. We can take advantage of this and use Gluon hybrid blocks and switch between hybrid and non-hybrid modes seamleessly. +MXNet currently provides three control flow operators: `cond`, `foreach` and `while_loop`. Like other MXNet operators, they all have a version for NDArray and a version for Symbol. These two versions have exactly the same semantics. We can take advantage of this and use them in Gluon to hybridize models. In this tutorial, we use a few examples to demonstrate the use of control flow operators in Gluon and show how a model that requires control flow is hybridized. @@ -18,13 +18,13 @@ from mxnet.gluon import HybridBlock foreach(body, data, init_states, name) => (outputs, states) ``` -It iterates over the first dimension of the input data (it can be an array or multiple arrays) and run the Python function defined in `body` for every slice from the input arrays. The signature of the `body` function is defined as follows: +It iterates over the first dimension of the input data (it can be an array or a list of arrays) and run the Python function defined in `body` for every slice from the input arrays. The signature of the `body` function is defined as follows: ```python body(data, states) => (outputs, states) ``` -The inputs of the `body` function have two parts: `data` is a slice of an array (if there is only one input array in `foreach`) or a list of slices (if there are multiple input arrays); `states` are the arrays from the previous iteration. The outputs of the `body` function also have two parts: `outputs` is an array or a list of arrays; `states` is the computation states of the current iteration. `outputs` from all iterations are concatenated as the outputs of `foreach`. +The inputs of the `body` function have two parts: `data` is a slice of an array (if there is only one input array in `foreach`) or a list of slices (if there are a list of input arrays); `states` are the arrays from the previous iteration. The outputs of the `body` function also have two parts: `outputs` is an array or a list of arrays; `states` is the computation states of the current iteration. `outputs` from all iterations are concatenated as the outputs of `foreach`. The pseudocode below illustrates the execution of `foreach`. @@ -42,7 +42,7 @@ def foreach(body, data, init_states): ``` ### Example 1: foreach works like map -`foreach` can work like a map function in a functional language. In this case, the states of foreach can be an empty list, which means the computation doesn't carry computation states across iterations. +`foreach` can work like a map function of a functional language. In this case, the states of foreach can be an empty list, which means the computation doesn't carry computation states across iterations. In this example, we use `foreach` to add each element in an array by one. @@ -336,7 +336,7 @@ class RNNLayer(HybridBlock): i = states[0] out, states = self.cell(i, length, data, states[1]) return out, [i + 1, states] - + print() out, state = F.contrib.foreach(body, data, [F.zeros((1)), init_states]) return out, state ``` @@ -360,23 +360,24 @@ print(rnn_data) print(out) ``` + () - [[[-0.70893967 0.79505837 1.01872194]] + [[[-1.25296438 0.387312 -0.41055229]] - [[-0.15738758 -1.5379014 -0.78309226]] + [[ 1.28453672 0.21001032 -0.08666432]] - [[ 0.5364728 -1.49940979 0.18817241]] + [[ 1.46422136 -1.30581355 0.9344402 ]] - [[-0.52421683 0.29636207 -0.53233677]] + [[ 0.5380863 -0.16038011 0.84187603]] - [[ 0.85569632 -0.87393355 -0.10898105]]] + [[-1.00553632 3.13221502 -0.4358989 ]]] - [[[ 0.32939154 -0.01365058 -0.25545752]] + [[[-0.02620504 0.1605694 0.29636264]] - [[ 0.15680683 -0.00335424 -0.14627317]] + [[-0.00474182 0.08719197 0.17757624]] - [[ 0.07623718 -0.00953147 -0.09720358]] + [[ 0.00631597 0.04674901 0.12468992]] [[ 0. 0. 0. ]] From bf5d333b1d24e0a1fc7da4d2682fb4d46b929b14 Mon Sep 17 00:00:00 2001 From: Da Zheng Date: Fri, 24 Aug 2018 15:44:41 -0700 Subject: [PATCH 03/11] add to test. --- tests/tutorials/test_tutorials.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/tutorials/test_tutorials.py b/tests/tutorials/test_tutorials.py index 2c8768228d71..b11297c071df 100644 --- a/tests/tutorials/test_tutorials.py +++ b/tests/tutorials/test_tutorials.py @@ -183,3 +183,6 @@ def test_vision_large_scale_classification(): def test_vision_cnn_visualization(): assert _test_tutorial_nb('vision/cnn_visualization') + +def test_control_flow(): + assert _test_control_flow('control_flow') From 1c3818d17b92cf6a5969285ade682abc26b2cfa8 Mon Sep 17 00:00:00 2001 From: Da Zheng Date: Fri, 24 Aug 2018 15:51:05 -0700 Subject: [PATCH 04/11] fix. --- tests/tutorials/test_tutorials.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tutorials/test_tutorials.py b/tests/tutorials/test_tutorials.py index b11297c071df..6f7bdd5c5d9d 100644 --- a/tests/tutorials/test_tutorials.py +++ b/tests/tutorials/test_tutorials.py @@ -185,4 +185,4 @@ def test_vision_cnn_visualization(): assert _test_tutorial_nb('vision/cnn_visualization') def test_control_flow(): - assert _test_control_flow('control_flow') + assert _test_control_flow('control_flow/ControlFlowTutorial') From 69e71a4c499ebae8ff1063d17924ec5cf6d64e3b Mon Sep 17 00:00:00 2001 From: Da Zheng Date: Fri, 24 Aug 2018 15:59:32 -0700 Subject: [PATCH 05/11] fix. --- tests/tutorials/test_tutorials.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tutorials/test_tutorials.py b/tests/tutorials/test_tutorials.py index 6f7bdd5c5d9d..503df017ffe0 100644 --- a/tests/tutorials/test_tutorials.py +++ b/tests/tutorials/test_tutorials.py @@ -185,4 +185,4 @@ def test_vision_cnn_visualization(): assert _test_tutorial_nb('vision/cnn_visualization') def test_control_flow(): - assert _test_control_flow('control_flow/ControlFlowTutorial') + assert _test_tutorial_nb('control_flow/ControlFlowTutorial') From f20c2006c815d1670040002f202abbe69b1618fe Mon Sep 17 00:00:00 2001 From: Da Zheng Date: Fri, 24 Aug 2018 16:13:25 -0700 Subject: [PATCH 06/11] fix --- docs/tutorials/control_flow/ControlFlowTutorial.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/tutorials/control_flow/ControlFlowTutorial.md b/docs/tutorials/control_flow/ControlFlowTutorial.md index ddc1f5f8d709..8e6dff6809a2 100644 --- a/docs/tutorials/control_flow/ControlFlowTutorial.md +++ b/docs/tutorials/control_flow/ControlFlowTutorial.md @@ -384,3 +384,5 @@ print(out) [[ 0. 0. 0. ]]] + + From d485eaa4ebbbad6485d7bfc59120db5b91d8fb4e Mon Sep 17 00:00:00 2001 From: Da Zheng Date: Fri, 24 Aug 2018 16:40:41 -0700 Subject: [PATCH 07/11] fix. --- docs/tutorials/control_flow/ControlFlowTutorial.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/tutorials/control_flow/ControlFlowTutorial.md b/docs/tutorials/control_flow/ControlFlowTutorial.md index 8e6dff6809a2..f0f42441490a 100644 --- a/docs/tutorials/control_flow/ControlFlowTutorial.md +++ b/docs/tutorials/control_flow/ControlFlowTutorial.md @@ -253,12 +253,12 @@ Instead of running over the first dimension of an array, `while_loop` checks a c body(state1, state2, ...) => (outputs, states) ``` -The inputs of the `body` function in `while_loop` are a little different from the one in `foreach`. It has a variable number of input arguments. Each input argument is a loop variable and the number of arguments is determined by the number of loop variables. The outputs of the `body` function also have two parts: `outputs` is an array or a list of arrays; `states` is the computation states of the current iteration. Like `foreach`, both `outputs` and `states` can be an empty list. `outputs` from all iterations are concatenated as the outputs of `while_loop`. +The inputs of the `body` function in `while_loop` are a little different from the one in `foreach`. It has a variable number of input arguments. Each input argument is a loop variable and the number of arguments is determined by the number of loop variables. The outputs of the `body` function also have two parts: `outputs` is an array or a list of arrays; `states` are loop variables and will be passed to the next iteration as inputs of `body`. Like `foreach`, both `outputs` and `states` can be an empty list. `outputs` from all iterations are concatenated as the outputs of `while_loop`. ### Example 5: scan with while_loop `while_loop` is more general than `foreach`. We can also use it to iterate over an array and sum all of its values together. In this example, instead of summing over the entire array, we only sum over the first 4 elements. -**Note**: the output arrays of the current implementation of `while_loop` is determined by `max_iterations`. As such, even though the while loop in this example runs 4 iterations, it still outputs an array of 5 elements. The last element in the output array is actually filled with a random number. +**Note**: the output arrays of the current implementation of `while_loop` is determined by `max_iterations`. As such, even though the while loop in this example runs 4 iterations, it still outputs an array of 5 elements. The last element in the output array is actually filled with an arbitrary value. ```python @@ -310,7 +310,9 @@ func() => [outputs] `cond` requires all outputs from `then_func` and `else_func` have the same number of Symbols/NDArrays with the same shapes and data types. ### Example 6: skip RNN computation with cond -Example 4 shows how to process a batch with sequences of different lengths. In this example, we show how to skip computation after we have met the end of the sequence. +Example 4 shows how to process a batch with sequences of different lengths. It performs computation for all steps but discards some of the computation results. + +In this example, we show how to skip computation after we have reached the end of a sequence, whose length is indicated by `length`. The code below only works for a batch with one sequence. ```python From 8e4dab02596dc5cadd992573ca254996f8b04679 Mon Sep 17 00:00:00 2001 From: Da Zheng Date: Sun, 26 Aug 2018 21:27:25 -0700 Subject: [PATCH 08/11] fix. --- .../control_flow/ControlFlowTutorial.md | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/docs/tutorials/control_flow/ControlFlowTutorial.md b/docs/tutorials/control_flow/ControlFlowTutorial.md index f0f42441490a..f7702df8503b 100644 --- a/docs/tutorials/control_flow/ControlFlowTutorial.md +++ b/docs/tutorials/control_flow/ControlFlowTutorial.md @@ -3,7 +3,7 @@ MXNet currently provides three control flow operators: `cond`, `foreach` and `wh In this tutorial, we use a few examples to demonstrate the use of control flow operators in Gluon and show how a model that requires control flow is hybridized. -# Prepare running the code +## Prepare running the code ```python @@ -11,14 +11,14 @@ import mxnet as mx from mxnet.gluon import HybridBlock ``` -# foreach +## foreach `foreach` is defined with the following signature: ```python foreach(body, data, init_states, name) => (outputs, states) ``` -It iterates over the first dimension of the input data (it can be an array or a list of arrays) and run the Python function defined in `body` for every slice from the input arrays. The signature of the `body` function is defined as follows: +It iterates over the first dimension of the input data (it can be an array or a list of arrays) and runs the Python function defined in `body` for every slice from the input arrays. The signature of the `body` function is defined as follows: ```python body(data, states) => (outputs, states) @@ -26,7 +26,7 @@ body(data, states) => (outputs, states) The inputs of the `body` function have two parts: `data` is a slice of an array (if there is only one input array in `foreach`) or a list of slices (if there are a list of input arrays); `states` are the arrays from the previous iteration. The outputs of the `body` function also have two parts: `outputs` is an array or a list of arrays; `states` is the computation states of the current iteration. `outputs` from all iterations are concatenated as the outputs of `foreach`. -The pseudocode below illustrates the execution of `foreach`. +The following pseudocode illustrates the execution of `foreach`. ```python def foreach(body, data, init_states): @@ -41,10 +41,10 @@ def foreach(body, data, init_states): return outs, states ``` -### Example 1: foreach works like map -`foreach` can work like a map function of a functional language. In this case, the states of foreach can be an empty list, which means the computation doesn't carry computation states across iterations. +### Example 1: `foreach` works like map +`foreach` can work like a map function of a functional language. In this case, the states of `foreach` can be an empty list, which means the computation doesn't carry computation states across iterations. -In this example, we use `foreach` to add each element in an array by one. +In this example, we use `foreach` to increase each element's value of an array by one. ```python @@ -99,7 +99,7 @@ print(out) -### Example 2: foreach works like scan +### Example 2: `foreach` works like scan `foreach` can work like a scan function in a functional language. In this case, the outputs of the Python function is an empty list. @@ -137,8 +137,8 @@ print(state) -### Example 3: foreach with both outputs and states -This is probably the most common use case of `foreach`. We extend the scan example above and return both output and states. +### Example 3: `foreach` with both outputs and states +This is probably the most common use case of `foreach`. We extend the previous scan example and return both output and states. ```python @@ -187,8 +187,8 @@ print(state) -### Example 4: use foreach to run RNN on a variable-length sequence -Previous examples illustrate `foreach` with simple use cases. Here I show an example of processing variable-length sequences with `foreach`. The same idea is used by dynamic_rnn in TensorFlow for processing variable-length sequences. +### Example 4: use `foreach` to run an RNN on a variable-length sequence +Previous examples illustrate `foreach` with simple use cases. Here we show an example of processing variable-length sequences with `foreach`. The same idea is used by `dynamic_rnn` in TensorFlow for processing variable-length sequences. ```python @@ -220,10 +220,8 @@ class DynamicRNNLayer(HybridBlock): use_sequence_length=True, axis=0) # the last state is the iteration number. We don't need it. return outputs, states[:-1] -``` -```python seq_len = 10 batch_size = 2 input_size = 5 @@ -241,7 +239,7 @@ lstm.hybridize() res, states = lstm(rnn_data, [x for x in init_states], valid_length) ``` -# while_loop +## while_loop `while_loop` is defined with the following signature: ```python @@ -294,7 +292,7 @@ print(state) ] -# cond +## cond `cond` is defined with the following signature: ```python @@ -341,10 +339,8 @@ class RNNLayer(HybridBlock): print() out, state = F.contrib.foreach(body, data, [F.zeros((1)), init_states]) return out, state -``` -```python seq_len = 5 batch_size = 1 input_size = 3 From 07d40dd527017cdfa838be18e76ec96e027081dd Mon Sep 17 00:00:00 2001 From: Da Zheng Date: Sun, 26 Aug 2018 21:40:44 -0700 Subject: [PATCH 09/11] add title. --- docs/tutorials/control_flow/ControlFlowTutorial.md | 1 + docs/tutorials/index.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/tutorials/control_flow/ControlFlowTutorial.md b/docs/tutorials/control_flow/ControlFlowTutorial.md index f7702df8503b..3d064f0607fc 100644 --- a/docs/tutorials/control_flow/ControlFlowTutorial.md +++ b/docs/tutorials/control_flow/ControlFlowTutorial.md @@ -1,3 +1,4 @@ +# Hybridize Gluon models with control flows. MXNet currently provides three control flow operators: `cond`, `foreach` and `while_loop`. Like other MXNet operators, they all have a version for NDArray and a version for Symbol. These two versions have exactly the same semantics. We can take advantage of this and use them in Gluon to hybridize models. diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index ae0851425be0..bc65d93bd5da 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -96,6 +96,7 @@ Select API:  * [Fine-Tuning a pre-trained ImageNet model with a new dataset](/faq/finetune.html) * [Large-Scale Multi-Host Multi-GPU Image Classification](/tutorials/vision/large_scale_classification.html) * [Importing an ONNX model into MXNet](/tutorials/onnx/super_resolution.html) + * []() * API Guides * Core APIs * NDArray From 00bb353f30cb7db797a5ee71e70c0ea2fc74d247 Mon Sep 17 00:00:00 2001 From: Da Zheng Date: Sun, 26 Aug 2018 21:41:56 -0700 Subject: [PATCH 10/11] add link --- docs/tutorials/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index bc65d93bd5da..1b32333bded6 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -96,7 +96,7 @@ Select API:  * [Fine-Tuning a pre-trained ImageNet model with a new dataset](/faq/finetune.html) * [Large-Scale Multi-Host Multi-GPU Image Classification](/tutorials/vision/large_scale_classification.html) * [Importing an ONNX model into MXNet](/tutorials/onnx/super_resolution.html) - * []() + * [Hybridize Gluon models with control flows](/tutorials/control_flow/ControlFlowTutorial.html) * API Guides * Core APIs * NDArray From 110e32c3ece9683b546f27cb53dc8d3ecb793cfb Mon Sep 17 00:00:00 2001 From: Da Zheng Date: Sun, 26 Aug 2018 21:49:18 -0700 Subject: [PATCH 11/11] fix. --- docs/tutorials/control_flow/ControlFlowTutorial.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/tutorials/control_flow/ControlFlowTutorial.md b/docs/tutorials/control_flow/ControlFlowTutorial.md index 3d064f0607fc..9e4c66f8521d 100644 --- a/docs/tutorials/control_flow/ControlFlowTutorial.md +++ b/docs/tutorials/control_flow/ControlFlowTutorial.md @@ -13,13 +13,13 @@ from mxnet.gluon import HybridBlock ``` ## foreach -`foreach` is defined with the following signature: +`foreach` is a for loop that iterates over the first dimension of the input data (it can be an array or a list of arrays). It is defined with the following signature: ```python foreach(body, data, init_states, name) => (outputs, states) ``` -It iterates over the first dimension of the input data (it can be an array or a list of arrays) and runs the Python function defined in `body` for every slice from the input arrays. The signature of the `body` function is defined as follows: +It runs the Python function defined in `body` for every slice from the input arrays. The signature of the `body` function is defined as follows: ```python body(data, states) => (outputs, states) @@ -241,11 +241,12 @@ res, states = lstm(rnn_data, [x for x in init_states], valid_length) ``` ## while_loop -`while_loop` is defined with the following signature: +`while_loop` defines a while loop. It has the following signature: ```python while_loop(cond, body, loop_vars, max_iterations, name) => (outputs, states) ``` + Instead of running over the first dimension of an array, `while_loop` checks a condition function in every iteration and runs a `body` function for computation. The signature of the `body` function is defined as follows: ```python @@ -294,7 +295,7 @@ print(state) ## cond -`cond` is defined with the following signature: +`cond` defines an if condition. It has the following signature: ```python cond(pred, then_func, else_func, name)