diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e233d17..5616e30 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,6 +4,7 @@ set(HEADER_FILES ./activation_function.hpp ./neural_net.hpp ./utility.hpp + ./forward_feed.hpp ) set(SOURCE_FILES ) diff --git a/src/cost_function.hpp b/src/cost_function.hpp new file mode 100644 index 0000000..e6e29a1 --- /dev/null +++ b/src/cost_function.hpp @@ -0,0 +1,16 @@ +#include +#include + +/** Categorical cross entropy loss function for multi category categorization + * tasks + * + */ +struct CategoricalCrossEntropy { + float static loss(std::vector y, std::vector yhat) { + float loss = 0; + for (int i; i < y.size(); i++) { + loss += y[i] * log(yhat[i]); + } + return loss; + } +}; diff --git a/src/forward_feed.hpp b/src/forward_feed.hpp new file mode 100644 index 0000000..068304c --- /dev/null +++ b/src/forward_feed.hpp @@ -0,0 +1,38 @@ +/** Apply forward feeding to a fully connect neural network. + * This struct stores the final output as well as the activations that occur + * for use in backpropagation + * + */ +#include "activation_function.hpp" +#include "matrix.hpp" +#include + +template struct ForwardFeed { + std::vector> m_activations; + std::vector m_yhat; + + ForwardFeed(const std::vector &x, + const std::vector> &weights) { + // Convert input vector to matrix + Matrix A = Matrix(x.size(), 1, x); + + // Feed each layer forward except the last layer using the user specified + // activation function + m_activations.reserve(weights.size()); + for (size_t i = 0; i < weights.size() - 1; i++) { + // Calculate Z = W * A + Matrix Z = weights[i] * A; + + // Apply activation function + ActivationFunction::apply(Z.data()); + m_activations.push_back(A); + A = Z; + } + + // Always use soft max for the final layer + Matrix Z = weights.back() * A; + SoftMax::apply(Z.data()); + + m_yhat = Z.data(); + }; +}; diff --git a/src/matrix.hpp b/src/matrix.hpp index 10362ed..64db1b7 100644 --- a/src/matrix.hpp +++ b/src/matrix.hpp @@ -13,7 +13,7 @@ public: Matrix(size_t rows, size_t cols, T value) : m_rows(rows), m_cols(cols), m_data(rows * cols, value) {} - // Create a matrix from a 1d vector using move semantics + // Create a matrix from a 1d vector Matrix(size_t rows, size_t cols, std::vector data) : m_rows(rows), m_cols(cols), m_data(data) { diff --git a/src/neural_net.hpp b/src/neural_net.hpp index 40759eb..d83d467 100644 --- a/src/neural_net.hpp +++ b/src/neural_net.hpp @@ -1,12 +1,11 @@ #ifndef NEURAL_NET_H #define NEURAL_NET_H -#include "activation_function.hpp" #include "matrix.hpp" #include #include -template class NeuralNet { +template class NeuralNet { public: NeuralNet(std::vector &layer_sizes) : m_sizes(layer_sizes) { // Create random sampling device @@ -59,40 +58,9 @@ public: m_weights = new_weights; }; - /** Pass input vector through the neural network. - * This is a fully connected neural network geometry. - * @param x Input vector - * @return output of feed forward phase - */ - std::vector feed_forward(const std::vector &x) { - // Convert input vector to matrix - Matrix A = Matrix(x.size(), 1, x); - - // Feed each layer forward except the last layer using the user specified - // activation function - for (size_t i = 0; i < m_sizes.size() - 2; i++) { - // Calculate Z = W * A - Matrix Z = m_weights[i] * A; - - // Apply activation function - ActivationFunction::apply(Z.data()); - A = Z; - } - - // Always use soft max for the final layer - Matrix Z = m_weights.back() * A; - SoftMax::apply(Z.data()); - - // Convert final output to vector - std::vector output(Z.rows()); - for (size_t i = 0; i < Z.rows(); i++) { - output[i] = Z(i, 0); - } - return output; - }; - private: std::vector m_sizes; std::vector> m_weights; }; + #endif diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt index a6bdde9..4fe0d4a 100644 --- a/tests/unit_tests/CMakeLists.txt +++ b/tests/unit_tests/CMakeLists.txt @@ -1,8 +1,8 @@ include_directories(${gtest_SOURCE_DIR}/include ${gtest_SOURCE_DIR}) set(TEST_SOURCES - test_activation_functions.cpp - test_neural_net.cpp + ./test_activation_functions.cpp + ./test_feed_forward.cpp ) add_executable(Unit_Tests_run diff --git a/tests/unit_tests/test_feed_forward.cpp b/tests/unit_tests/test_feed_forward.cpp new file mode 100644 index 0000000..e21195e --- /dev/null +++ b/tests/unit_tests/test_feed_forward.cpp @@ -0,0 +1,134 @@ +#include "../../src/activation_function.hpp" +#include "../../src/forward_feed.hpp" +#include "../../src/matrix.hpp" +#include +#include +#include + +class ForwardFeedTest : public ::testing::Test { +protected: + void SetUp() override { + // Create simple weights for testing + weights = {Matrix(2, 2, {0.5, 0.5, 0.5, 0.5}), + Matrix(2, 2, {0.5, 0.5, 0.5, 0.5})}; + } + + std::vector> weights; +}; + +TEST_F(ForwardFeedTest, BasicForwardFeed) { + + // Create input data + std::vector input = {1.0, 2.0}; + + // Create ForwardFeed with ReLU activation + ForwardFeed feed(input, weights); + + // Verify output size + EXPECT_EQ(feed.m_yhat.size(), 2); + + // Verify number of activations stored + EXPECT_EQ(feed.m_activations.size(), 1); // Only one hidden layer + + // Verify input was stored as first activation + EXPECT_EQ(feed.m_activations[0].rows(), 2); + EXPECT_EQ(feed.m_activations[0].cols(), 1); + EXPECT_FLOAT_EQ(feed.m_activations[0](0, 0), 1.0); + EXPECT_FLOAT_EQ(feed.m_activations[0](1, 0), 2.0); +} + +TEST_F(ForwardFeedTest, DifferentActivationFunctions) { + // Test with different activation functions + std::vector input = {1.0, 2.0}; + + // Test with Sigmoid + ForwardFeed sigmoid_feed(input, weights); + EXPECT_EQ(sigmoid_feed.m_yhat.size(), 2); + + // Test with ReLU + ForwardFeed relu_feed(input, weights); + EXPECT_EQ(relu_feed.m_yhat.size(), 2); + + // Test with different input values + std::vector neg_input = {-1.0, -2.0}; + ForwardFeed neg_feed(neg_input, weights); + EXPECT_EQ(neg_feed.m_yhat.size(), 2); +} + +TEST_F(ForwardFeedTest, ActivationStorage) { + // Test that activations are properly stored + std::vector input = {1.0, 2.0}; + ForwardFeed feed(input, weights); + + // Verify first activation (input) + EXPECT_EQ(feed.m_activations[0].rows(), 2); + EXPECT_EQ(feed.m_activations[0].cols(), 1); + EXPECT_FLOAT_EQ(feed.m_activations[0](0, 0), 1.0); + EXPECT_FLOAT_EQ(feed.m_activations[0](1, 0), 2.0); + + // Verify final output (after softmax) + EXPECT_EQ(feed.m_yhat.size(), 2); + float sum = 0.0; + for (float val : feed.m_yhat) { + sum += val; + } + EXPECT_NEAR(sum, 1.0, 1e-6); // Softmax outputs should sum to 1 +} + +TEST_F(ForwardFeedTest, EdgeCases) { + // Test with zero input + std::vector zero_input = {0.0, 0.0}; + ForwardFeed zero_feed(zero_input, weights); + EXPECT_EQ(zero_feed.m_yhat.size(), 2); + + // Test with negative input + std::vector neg_input = {-1.0, -2.0}; + ForwardFeed neg_feed(neg_input, weights); + EXPECT_EQ(neg_feed.m_yhat.size(), 2); + + // Test with large input + std::vector large_input = {100.0, 200.0}; + ForwardFeed large_feed(large_input, weights); + EXPECT_EQ(large_feed.m_yhat.size(), 2); +} + +TEST_F(ForwardFeedTest, DifferentNetworkSizes) { + // Test with different network architectures + std::vector>> test_weights = { + // Single hidden layer + {Matrix(2, 2, {0.5, 0.5, 0.5, 0.5}), + Matrix(2, 2, {0.5, 0.5, 0.5, 0.5})}, + // Multiple hidden layers + {Matrix(3, 2, {0.5, 0.5, 0.5, 0.5, 0.5, 0.5}), + Matrix(2, 3, {0.5, 0.5, 0.5, 0.5, 0.5, 0.5}), + Matrix(2, 2, {0.5, 0.5, 0.5, 0.5})}}; + + for (const auto &w : test_weights) { + std::vector input(2, 1.0); + ForwardFeed feed(input, w); + + // Verify number of activations matches number of hidden layers + EXPECT_EQ(feed.m_activations.size(), w.size() - 1); + + // Verify final output size + EXPECT_EQ(feed.m_yhat.size(), w.back().rows()); + } +} + +TEST_F(ForwardFeedTest, WeightMatrixDimensions) { + // Test with invalid weight matrix dimensions + std::vector input = {1.0, 2.0}; + + // Test with mismatched dimensions + std::vector> invalid_weights = { + Matrix(2, 3, {0.5, 0.5, 0.5, 0.5, 0.5, 0.5}), // 3x2 instead of 2x2 + Matrix(2, 2, {0.5, 0.5, 0.5, 0.5})}; + + EXPECT_THROW(ForwardFeed feed(input, invalid_weights), + std::invalid_argument); +} + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/unit_tests/test_neural_net.cpp b/tests/unit_tests/test_neural_net.cpp deleted file mode 100644 index f74d83b..0000000 --- a/tests/unit_tests/test_neural_net.cpp +++ /dev/null @@ -1,119 +0,0 @@ -#include "../src/activation_function.hpp" -#include "../src/neural_net.hpp" -#include -#include -#include - -class NeuralNetTest : public ::testing::Test { -protected: - void SetUp() override { - // Create a simple neural network with 2 input neurons, 2 hidden neurons, - // and 2 output neurons - std::vector layer_sizes = {2, 2, 2}; - net = std::make_unique>(layer_sizes); - } - - std::unique_ptr> net; -}; - -TEST_F(NeuralNetTest, FeedForward_SimpleNetwork) { - // Test a simple network with known weights and inputs - std::vector input = {0.5f, 0.5f}; - - // Set known weights for testing - std::vector> weights = { - Matrix(2, 2, 0.5f), // First layer weights - Matrix(2, 2, 0.5f) // Output layer weights - }; - - // Replace the network's weights with our test weights - net->set_weights(weights); - - // Calculate expected output manually - // First layer: Z1 = W1 * X - Matrix X(2, 1, 0.0); - X(0, 0) = input[0]; - X(1, 0) = input[1]; - - Matrix Z1 = weights[0] * X; - // Apply sigmoid activation - Sigmoid::apply(Z1.data()); - - // Second layer: Z2 = W2 * A1 - Matrix Z2 = weights[1] * Z1; - SoftMax::apply(Z2.data()); - - // Convert to output vector - std::vector expected_output(Z2.cols()); - for (size_t i = 0; i < Z2.rows(); i++) { - expected_output[i] = Z2(i, 0); - } - - // Get actual output from feed_forward - std::vector output = net->feed_forward(input); - - // Compare actual and expected outputs - for (size_t i = 0; i < output.size(); i++) { - EXPECT_NEAR(output[i], expected_output[i], 1e-6); - } -} - -TEST_F(NeuralNetTest, FeedForward_DifferentLayerSizes) { - // Create a network with different layer sizes - std::vector layer_sizes = {3, 4, 2}; - NeuralNet net2(layer_sizes); - - std::vector input = {0.1f, 0.2f, 0.3f}; - std::vector output = net2.feed_forward(input); - - // Output should have 2 elements (size of last layer) - EXPECT_EQ(output.size(), 2); -} - -TEST_F(NeuralNetTest, FeedForward_InvalidInputSize) { - std::vector input = {0.1f}; // Only 1 input, but network expects 2 - - // This should throw an exception since input size doesn't match first layer - // size - EXPECT_THROW(net->feed_forward(input), std::invalid_argument); -} - -TEST_F(NeuralNetTest, FeedForward_IdentityTest) { - // Create a network with identity weights (1.0) and no bias - std::vector layer_sizes = {2, 2}; - NeuralNet net2(layer_sizes); - - // Set weights to identity matrix - std::vector> weights = {Matrix(2, 2, 1.0f)}; - - net2.set_weights(weights); - - std::vector input = {0.5f, 0.5f}; - std::vector output = net2.feed_forward(input); - - // Since we're using sigmoid activation, the output should be - // sigmoid(0.5 + 0.5) = sigmoid(1.0) for each neuron - std::vector expected_output = input; - SoftMax::apply(expected_output); - - for (float val : output) { - EXPECT_NEAR(val, expected_output[0], 1e-6); - } -} - -TEST_F(NeuralNetTest, FeedForward_SoftmaxOutput) { - std::vector input = {1.0f, -1.0f}; - std::vector output = net->feed_forward(input); - - // Verify that the output sums to 1 (property of softmax) - float sum = 0.0f; - for (float val : output) { - sum += val; - } - EXPECT_NEAR(sum, 1.0f, 1e-6); - - // Verify that all outputs are positive - for (float val : output) { - EXPECT_GT(val, 0.0f); - } -}