Skip to content

Commit 2926dc1

Browse files
fm94fm94
committed
xor test - intermediate state
1 parent a800c8f commit 2926dc1

File tree

11 files changed

+123
-24
lines changed

11 files changed

+123
-24
lines changed

Diff for: ‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
.vscode/
22
.vs/
33
build/
4+
CMakeSettings.json

Diff for: ‎include/data/dataset.hpp

+6-6
Original file line numberDiff line numberDiff line change
@@ -9,32 +9,32 @@ namespace Tipousi
99
class Dataset
1010
{
1111
public:
12-
Dataset(const Eigen::MatrixXd &X, const Eigen::MatrixXd &Y);
12+
Dataset(const Eigen::MatrixXf &X, const Eigen::MatrixXf &Y);
1313
~Dataset() = default;
1414

1515
using DataPair = std::pair<Eigen::MatrixXf, Eigen::MatrixXf>;
1616

1717
class Iterator
1818
{
1919
public:
20-
Iterator(const Eigen::MatrixXd &X, const Eigen::MatrixXd &Y,
20+
Iterator(const Eigen::MatrixXf &X, const Eigen::MatrixXf &Y,
2121
size_t index);
2222
Iterator &operator++();
2323
bool operator!=(const Iterator &other) const;
2424
DataPair operator*() const;
2525

2626
private:
27-
const Eigen::MatrixXd &m_X;
28-
const Eigen::MatrixXd &m_y;
27+
const Eigen::MatrixXf &m_X;
28+
const Eigen::MatrixXf &m_y;
2929
size_t m_index;
3030
};
3131

3232
Iterator begin() const;
3333
Iterator end() const;
3434

3535
private:
36-
Eigen::MatrixXd m_X;
37-
Eigen::MatrixXd m_y;
36+
Eigen::MatrixXf m_X;
37+
Eigen::MatrixXf m_y;
3838
};
3939
}; // namespace Data
4040
}; // namespace Tipousi

Diff for: ‎include/graph/node.hpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ namespace Tipousi
1919
}
2020

2121
void forward(Eigen::MatrixXf &data);
22-
void backward(const Eigen::MatrixXf &dout, Eigen::MatrixXf &ddout);
22+
void backward(Eigen::MatrixXf &grads);
2323

2424
void add_input(Node *node);
2525
void add_output(Node *node);

Diff for: ‎include/graph/sequential.hpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ namespace Tipousi
2626
}
2727

2828
void forward(const Eigen::MatrixXf &in, Eigen::MatrixXf &out);
29-
void backward();
29+
void backward(Eigen::MatrixXf &initial_grads);
3030

3131
void train(const Data::Dataset &dataset,
3232
const Optimizer::OptimizerBase &optimizer,

Diff for: ‎include/loss/mse.hpp

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ namespace Tipousi
77
{
88
class MSE : public LossBase
99
{
10+
public:
1011
MSE() = default;
1112
~MSE() = default;
1213

Diff for: ‎src/data/dataset.cpp

+3-3
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,13 @@ namespace Tipousi
44
{
55
namespace Data
66
{
7-
Dataset::Dataset(const Eigen::MatrixXd &X, const Eigen::MatrixXd &Y)
7+
Dataset::Dataset(const Eigen::MatrixXf &X, const Eigen::MatrixXf &Y)
88
: m_X(X), m_y(Y)
99
{
1010
}
1111

12-
Dataset::Iterator::Iterator(const Eigen::MatrixXd &X,
13-
const Eigen::MatrixXd &Y, size_t index)
12+
Dataset::Iterator::Iterator(const Eigen::MatrixXf &X,
13+
const Eigen::MatrixXf &Y, size_t index)
1414
: m_X(X), m_y(Y), m_index(index)
1515
{
1616
}

Diff for: ‎src/graph/node.cpp

+5-5
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,13 @@ namespace Tipousi
2020

2121
// if we have mutliple inputs then save forward computations by
2222
// caching the current data. Experimental!!!
23-
if (m_outputs.size() > 1)
24-
{
23+
//if (m_outputs.size() > 1)
24+
// {
2525
// m_cache
26-
}
26+
// }
2727
}
2828

29-
void Node::backward(const Eigen::MatrixXf &dout, Eigen::MatrixXf &ddout)
29+
void Node::backward(Eigen::MatrixXf &grads)
3030
{
3131
// std::vector<float> grad_input =
3232
// m_operation->backward(grad_output); for (auto *input_node :
@@ -36,7 +36,7 @@ namespace Tipousi
3636
// }
3737

3838
// TODO this is a fake call that is used only if the node has one Op
39-
m_operation->backward(dout, ddout);
39+
m_operation->backward(grads, grads);
4040
}
4141

4242
void Node::add_input(Node *node)

Diff for: ‎src/graph/sequential.cpp

+22-3
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ namespace Tipousi
3232
void Sequential::forward(const Eigen::MatrixXf &in,
3333
Eigen::MatrixXf &out)
3434
{
35+
// in should be const?
36+
//
3537
// copy to create the object that will be passed through the network
3638
// while keeping the original data intact
3739
Eigen::MatrixXf data_copy = in;
@@ -54,9 +56,26 @@ namespace Tipousi
5456
out = data_copy; // copying happening?
5557
}
5658

57-
void Sequential::backward()
59+
void Sequential::backward(Eigen::MatrixXf &initial_grads)
5860
{
59-
//
61+
// think if copying is really needed here
62+
Eigen::MatrixXf grad_copy = initial_grads;
63+
Node *current_node = m_output_node;
64+
// continue until no more nodes
65+
while (true)
66+
{
67+
if (current_node)
68+
{
69+
// TODO hacky approachs: always take number 0
70+
auto &input_nodes = current_node->get_inputs();
71+
if (input_nodes.size() == 0 || !input_nodes[0])
72+
{
73+
break;
74+
}
75+
current_node->backward(grad_copy);
76+
current_node = input_nodes[0];
77+
}
78+
}
6079
}
6180

6281
void Sequential::train(const Data::Dataset &dataset,
@@ -75,7 +94,7 @@ namespace Tipousi
7594
forward(x, output);
7695
total_loss += loss_func.compute(y, output);
7796
loss_func.grad(out_grad, y, output);
78-
backward();
97+
backward(out_grad);
7998
counter++;
8099
}
81100
std::cout << "Epoch: " << i

Diff for: ‎tests/test_simple_net.cpp

+5-3
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,11 @@ TEST(SimpleNetTest, SimpleCreation)
3232
node4->add_input(node3); // node4 depends on node2
3333

3434
// create the graph (pass input and output nodes)
35-
float learning_rate{0.001};
35+
float learning_rate{0.001f};
3636
Sequential net(node1, node4, learning_rate);
3737

3838
// test inference
39-
int n_samples{32};
39+
int n_samples{10};
4040
auto features = Eigen::MatrixXf::Random(n_samples, n_features);
4141
auto labels = Eigen::MatrixXf(n_samples, n_labels);
4242

@@ -53,5 +53,7 @@ TEST(SimpleNetTest, SimpleCreation)
5353
<< std::endl;
5454

5555
// backward pass
56-
EXPECT_NO_THROW(net.backward());
56+
// todo: add real values here, can't do backward pass with empty matrix
57+
// Eigen::MatrixXf out_grad;
58+
// EXPECT_NO_THROW(net.backward(out_grad));
5759
}

Diff for: ‎tests/test_softmax.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ TEST(SoftmaxLayerTest, ForwardPass)
1717
Eigen::MatrixXf expected_output(2, 2);
1818
expected_output << 0.268941f, 0.731059f, 0.268941f, 0.731059f;
1919

20-
expectEigenMatrixNear(output, expected_output, 1e-5);
20+
expectEigenMatrixNear(output, expected_output, 1e-5f);
2121
}
2222

2323
TEST(SoftmaxLayerTest, BackwardPass)
@@ -39,5 +39,5 @@ TEST(SoftmaxLayerTest, BackwardPass)
3939
Eigen::MatrixXf expected_in_grad(2, 2);
4040
expected_in_grad << 0.0393f, -0.0393f, 0.0393f, -0.0393f;
4141

42-
expectEigenMatrixNear(in_grad, expected_in_grad, 1e-3);
42+
expectEigenMatrixNear(in_grad, expected_in_grad, 1e-3f);
4343
}

Diff for: ‎tests/test_xor_inference.cpp

+76
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
#include "activation/relu.hpp"
2+
#include "activation/softmax.hpp"
3+
#include "data/dataset.hpp"
4+
#include "graph/node.hpp"
5+
#include "graph/sequential.hpp"
6+
#include "layer/dense.hpp"
7+
#include "loss/mse.hpp"
8+
#include "optimizer/sgd.hpp"
9+
#include <chrono>
10+
#include <gtest/gtest.h>
11+
#include <iostream>
12+
#include <memory>
13+
14+
using namespace Tipousi;
15+
using namespace Graph;
16+
using namespace Layer;
17+
using namespace Activation;
18+
using namespace Loss;
19+
using namespace Data;
20+
using namespace Optimizer;
21+
22+
TEST(SimpleNetTest, XORTest)
23+
{
24+
// in this test we try to train a net to lear the xor operation
25+
// we have two inputs and one output
26+
int n_features{2};
27+
int n_labels{1};
28+
29+
Node *node1 = Node::create<Dense>(n_features, 32);
30+
Node *node2 = Node::create<ReLU>();
31+
Node *node3 = Node::create<Dense>(32, n_labels);
32+
Node *node4 = Node::create<Softmax>();
33+
34+
// build the dependencies
35+
node2->add_input(node1); // node2 depends on node1
36+
node3->add_input(node2); // node3 depends on node2
37+
node4->add_input(node3); // node4 depends on node2
38+
39+
// create the graph (pass input and output nodes)
40+
float learning_rate{0.001f};
41+
Sequential net(node1, node4, learning_rate);
42+
43+
// test inference
44+
Eigen::MatrixXf X(4, 2);
45+
Eigen::MatrixXf Y(4, 1);
46+
// XOR inputs
47+
X << 0, 0, 0, 1, 1, 0, 1, 1;
48+
// XOR outputs (labels)
49+
Y << 0, 1, 1, 0;
50+
51+
Eigen::MatrixXf preds;
52+
net.forward(X, preds);
53+
54+
// create dataset
55+
// TODO add eppochs etc
56+
Dataset dataset(X, Y);
57+
58+
// define the optimizer and the loss
59+
SGD sgd;
60+
MSE mse;
61+
net.train(dataset, sgd, mse, 10);
62+
63+
// forward pass with time measurement
64+
// Eigen::MatrixXf preds;
65+
auto start = std::chrono::high_resolution_clock::now();
66+
net.forward(X, preds);
67+
auto end = std::chrono::high_resolution_clock::now();
68+
auto duration =
69+
std::chrono::duration_cast<std::chrono::microseconds>(end - start)
70+
.count();
71+
std::cout << "Forward pass execution time: " << duration << " microseconds"
72+
<< std::endl;
73+
74+
// compute the loss
75+
std::cout << "The loss of the network after training: " << mse.compute(Y, preds) << std::endl;
76+
}

0 commit comments

Comments
 (0)