Refactor forward propagation code again

This commit is contained in:
Alex Selimov 2025-03-30 00:40:12 -04:00
parent 47ef7c25d7
commit 92e9a4a719
8 changed files with 194 additions and 156 deletions

View File

@ -4,6 +4,7 @@ set(HEADER_FILES
./activation_function.hpp
./neural_net.hpp
./utility.hpp
./forward_feed.hpp
)
set(SOURCE_FILES
)

16
src/cost_function.hpp Normal file
View File

@ -0,0 +1,16 @@
#include <cmath>
#include <vector>
/** Categorical cross entropy loss function for multi category categorization
* tasks
*
*/
struct CategoricalCrossEntropy {
float static loss(std::vector<float> y, std::vector<float> yhat) {
float loss = 0;
for (int i; i < y.size(); i++) {
loss += y[i] * log(yhat[i]);
}
return loss;
}
};

38
src/forward_feed.hpp Normal file
View File

@ -0,0 +1,38 @@
/** Apply forward feeding to a fully connect neural network.
* This struct stores the final output as well as the activations that occur
* for use in backpropagation
*
*/
#include "activation_function.hpp"
#include "matrix.hpp"
#include <vector>
template <class ActivationFunction> struct ForwardFeed {
std::vector<Matrix<float>> m_activations;
std::vector<float> m_yhat;
ForwardFeed(const std::vector<float> &x,
const std::vector<Matrix<float>> &weights) {
// Convert input vector to matrix
Matrix<float> A = Matrix<float>(x.size(), 1, x);
// Feed each layer forward except the last layer using the user specified
// activation function
m_activations.reserve(weights.size());
for (size_t i = 0; i < weights.size() - 1; i++) {
// Calculate Z = W * A
Matrix Z = weights[i] * A;
// Apply activation function
ActivationFunction::apply(Z.data());
m_activations.push_back(A);
A = Z;
}
// Always use soft max for the final layer
Matrix Z = weights.back() * A;
SoftMax::apply(Z.data());
m_yhat = Z.data();
};
};

View File

@ -13,7 +13,7 @@ public:
Matrix(size_t rows, size_t cols, T value)
: m_rows(rows), m_cols(cols), m_data(rows * cols, value) {}
// Create a matrix from a 1d vector using move semantics
// Create a matrix from a 1d vector
Matrix(size_t rows, size_t cols, std::vector<T> data)
: m_rows(rows), m_cols(cols), m_data(data) {

View File

@ -1,12 +1,11 @@
#ifndef NEURAL_NET_H
#define NEURAL_NET_H
#include "activation_function.hpp"
#include "matrix.hpp"
#include <random>
#include <vector>
template <class ActivationFunction> class NeuralNet {
template <class ActivationFunction, class LossFunction> class NeuralNet {
public:
NeuralNet(std::vector<size_t> &layer_sizes) : m_sizes(layer_sizes) {
// Create random sampling device
@ -59,40 +58,9 @@ public:
m_weights = new_weights;
};
/** Pass input vector through the neural network.
* This is a fully connected neural network geometry.
* @param x Input vector
* @return output of feed forward phase
*/
std::vector<float> feed_forward(const std::vector<float> &x) {
// Convert input vector to matrix
Matrix<float> A = Matrix<float>(x.size(), 1, x);
// Feed each layer forward except the last layer using the user specified
// activation function
for (size_t i = 0; i < m_sizes.size() - 2; i++) {
// Calculate Z = W * A
Matrix Z = m_weights[i] * A;
// Apply activation function
ActivationFunction::apply(Z.data());
A = Z;
}
// Always use soft max for the final layer
Matrix Z = m_weights.back() * A;
SoftMax::apply(Z.data());
// Convert final output to vector
std::vector<float> output(Z.rows());
for (size_t i = 0; i < Z.rows(); i++) {
output[i] = Z(i, 0);
}
return output;
};
private:
std::vector<size_t> m_sizes;
std::vector<Matrix<float>> m_weights;
};
#endif

View File

@ -1,8 +1,8 @@
include_directories(${gtest_SOURCE_DIR}/include ${gtest_SOURCE_DIR})
set(TEST_SOURCES
test_activation_functions.cpp
test_neural_net.cpp
./test_activation_functions.cpp
./test_feed_forward.cpp
)
add_executable(Unit_Tests_run

View File

@ -0,0 +1,134 @@
#include "../../src/activation_function.hpp"
#include "../../src/forward_feed.hpp"
#include "../../src/matrix.hpp"
#include <gtest/gtest.h>
#include <stdexcept>
#include <vector>
class ForwardFeedTest : public ::testing::Test {
protected:
void SetUp() override {
// Create simple weights for testing
weights = {Matrix<float>(2, 2, {0.5, 0.5, 0.5, 0.5}),
Matrix<float>(2, 2, {0.5, 0.5, 0.5, 0.5})};
}
std::vector<Matrix<float>> weights;
};
TEST_F(ForwardFeedTest, BasicForwardFeed) {
// Create input data
std::vector<float> input = {1.0, 2.0};
// Create ForwardFeed with ReLU activation
ForwardFeed<ReLU> feed(input, weights);
// Verify output size
EXPECT_EQ(feed.m_yhat.size(), 2);
// Verify number of activations stored
EXPECT_EQ(feed.m_activations.size(), 1); // Only one hidden layer
// Verify input was stored as first activation
EXPECT_EQ(feed.m_activations[0].rows(), 2);
EXPECT_EQ(feed.m_activations[0].cols(), 1);
EXPECT_FLOAT_EQ(feed.m_activations[0](0, 0), 1.0);
EXPECT_FLOAT_EQ(feed.m_activations[0](1, 0), 2.0);
}
TEST_F(ForwardFeedTest, DifferentActivationFunctions) {
// Test with different activation functions
std::vector<float> input = {1.0, 2.0};
// Test with Sigmoid
ForwardFeed<Sigmoid> sigmoid_feed(input, weights);
EXPECT_EQ(sigmoid_feed.m_yhat.size(), 2);
// Test with ReLU
ForwardFeed<ReLU> relu_feed(input, weights);
EXPECT_EQ(relu_feed.m_yhat.size(), 2);
// Test with different input values
std::vector<float> neg_input = {-1.0, -2.0};
ForwardFeed<ReLU> neg_feed(neg_input, weights);
EXPECT_EQ(neg_feed.m_yhat.size(), 2);
}
TEST_F(ForwardFeedTest, ActivationStorage) {
// Test that activations are properly stored
std::vector<float> input = {1.0, 2.0};
ForwardFeed<ReLU> feed(input, weights);
// Verify first activation (input)
EXPECT_EQ(feed.m_activations[0].rows(), 2);
EXPECT_EQ(feed.m_activations[0].cols(), 1);
EXPECT_FLOAT_EQ(feed.m_activations[0](0, 0), 1.0);
EXPECT_FLOAT_EQ(feed.m_activations[0](1, 0), 2.0);
// Verify final output (after softmax)
EXPECT_EQ(feed.m_yhat.size(), 2);
float sum = 0.0;
for (float val : feed.m_yhat) {
sum += val;
}
EXPECT_NEAR(sum, 1.0, 1e-6); // Softmax outputs should sum to 1
}
TEST_F(ForwardFeedTest, EdgeCases) {
// Test with zero input
std::vector<float> zero_input = {0.0, 0.0};
ForwardFeed<ReLU> zero_feed(zero_input, weights);
EXPECT_EQ(zero_feed.m_yhat.size(), 2);
// Test with negative input
std::vector<float> neg_input = {-1.0, -2.0};
ForwardFeed<ReLU> neg_feed(neg_input, weights);
EXPECT_EQ(neg_feed.m_yhat.size(), 2);
// Test with large input
std::vector<float> large_input = {100.0, 200.0};
ForwardFeed<ReLU> large_feed(large_input, weights);
EXPECT_EQ(large_feed.m_yhat.size(), 2);
}
TEST_F(ForwardFeedTest, DifferentNetworkSizes) {
// Test with different network architectures
std::vector<std::vector<Matrix<float>>> test_weights = {
// Single hidden layer
{Matrix<float>(2, 2, {0.5, 0.5, 0.5, 0.5}),
Matrix<float>(2, 2, {0.5, 0.5, 0.5, 0.5})},
// Multiple hidden layers
{Matrix<float>(3, 2, {0.5, 0.5, 0.5, 0.5, 0.5, 0.5}),
Matrix<float>(2, 3, {0.5, 0.5, 0.5, 0.5, 0.5, 0.5}),
Matrix<float>(2, 2, {0.5, 0.5, 0.5, 0.5})}};
for (const auto &w : test_weights) {
std::vector<float> input(2, 1.0);
ForwardFeed<ReLU> feed(input, w);
// Verify number of activations matches number of hidden layers
EXPECT_EQ(feed.m_activations.size(), w.size() - 1);
// Verify final output size
EXPECT_EQ(feed.m_yhat.size(), w.back().rows());
}
}
TEST_F(ForwardFeedTest, WeightMatrixDimensions) {
// Test with invalid weight matrix dimensions
std::vector<float> input = {1.0, 2.0};
// Test with mismatched dimensions
std::vector<Matrix<float>> invalid_weights = {
Matrix<float>(2, 3, {0.5, 0.5, 0.5, 0.5, 0.5, 0.5}), // 3x2 instead of 2x2
Matrix<float>(2, 2, {0.5, 0.5, 0.5, 0.5})};
EXPECT_THROW(ForwardFeed<ReLU> feed(input, invalid_weights),
std::invalid_argument);
}
int main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

View File

@ -1,119 +0,0 @@
#include "../src/activation_function.hpp"
#include "../src/neural_net.hpp"
#include <cmath>
#include <gtest/gtest.h>
#include <stdexcept>
class NeuralNetTest : public ::testing::Test {
protected:
void SetUp() override {
// Create a simple neural network with 2 input neurons, 2 hidden neurons,
// and 2 output neurons
std::vector<size_t> layer_sizes = {2, 2, 2};
net = std::make_unique<NeuralNet<Sigmoid>>(layer_sizes);
}
std::unique_ptr<NeuralNet<Sigmoid>> net;
};
TEST_F(NeuralNetTest, FeedForward_SimpleNetwork) {
// Test a simple network with known weights and inputs
std::vector<float> input = {0.5f, 0.5f};
// Set known weights for testing
std::vector<Matrix<float>> weights = {
Matrix<float>(2, 2, 0.5f), // First layer weights
Matrix<float>(2, 2, 0.5f) // Output layer weights
};
// Replace the network's weights with our test weights
net->set_weights(weights);
// Calculate expected output manually
// First layer: Z1 = W1 * X
Matrix<float> X(2, 1, 0.0);
X(0, 0) = input[0];
X(1, 0) = input[1];
Matrix<float> Z1 = weights[0] * X;
// Apply sigmoid activation
Sigmoid::apply(Z1.data());
// Second layer: Z2 = W2 * A1
Matrix<float> Z2 = weights[1] * Z1;
SoftMax::apply(Z2.data());
// Convert to output vector
std::vector<float> expected_output(Z2.cols());
for (size_t i = 0; i < Z2.rows(); i++) {
expected_output[i] = Z2(i, 0);
}
// Get actual output from feed_forward
std::vector<float> output = net->feed_forward(input);
// Compare actual and expected outputs
for (size_t i = 0; i < output.size(); i++) {
EXPECT_NEAR(output[i], expected_output[i], 1e-6);
}
}
TEST_F(NeuralNetTest, FeedForward_DifferentLayerSizes) {
// Create a network with different layer sizes
std::vector<size_t> layer_sizes = {3, 4, 2};
NeuralNet<Sigmoid> net2(layer_sizes);
std::vector<float> input = {0.1f, 0.2f, 0.3f};
std::vector<float> output = net2.feed_forward(input);
// Output should have 2 elements (size of last layer)
EXPECT_EQ(output.size(), 2);
}
TEST_F(NeuralNetTest, FeedForward_InvalidInputSize) {
std::vector<float> input = {0.1f}; // Only 1 input, but network expects 2
// This should throw an exception since input size doesn't match first layer
// size
EXPECT_THROW(net->feed_forward(input), std::invalid_argument);
}
TEST_F(NeuralNetTest, FeedForward_IdentityTest) {
// Create a network with identity weights (1.0) and no bias
std::vector<size_t> layer_sizes = {2, 2};
NeuralNet<Sigmoid> net2(layer_sizes);
// Set weights to identity matrix
std::vector<Matrix<float>> weights = {Matrix<float>(2, 2, 1.0f)};
net2.set_weights(weights);
std::vector<float> input = {0.5f, 0.5f};
std::vector<float> output = net2.feed_forward(input);
// Since we're using sigmoid activation, the output should be
// sigmoid(0.5 + 0.5) = sigmoid(1.0) for each neuron
std::vector<float> expected_output = input;
SoftMax::apply(expected_output);
for (float val : output) {
EXPECT_NEAR(val, expected_output[0], 1e-6);
}
}
TEST_F(NeuralNetTest, FeedForward_SoftmaxOutput) {
std::vector<float> input = {1.0f, -1.0f};
std::vector<float> output = net->feed_forward(input);
// Verify that the output sums to 1 (property of softmax)
float sum = 0.0f;
for (float val : output) {
sum += val;
}
EXPECT_NEAR(sum, 1.0f, 1e-6);
// Verify that all outputs are positive
for (float val : output) {
EXPECT_GT(val, 0.0f);
}
}