WIP implementation of feed forward

CMakeLists.txt

@@ -1,6 +1,8 @@
 cmake_minimum_required(VERSION 3.9)
 project(NeuralNet)
 
+add_compile_options(-Wall -Wextra -Wpedantic)
+
 set(CMAKE_CXX_STANDARD 17)
 
 set(SOURCE_FILES main.cpp)

src/CMakeLists.txt

@@ -1,13 +1,17 @@
 project(${CMAKE_PROJECT_NAME}_lib)
 
 set(HEADER_FILES
-    activation_function.hpp
-    neural_net.hpp
+    ./activation_function.hpp
+    ./neural_net.hpp
+    ./utility.hpp
 )
 set(SOURCE_FILES
+    ./neural_net.cpp
 )
 
-if (EXISTS ${SOURCE_FILES})
+# Check if any source files exist
+list(LENGTH SOURCE_FILES SOURCE_FILES_LENGTH)
+if (SOURCE_FILES_LENGTH GREATER 0)
     # The library contains header and source files.
     add_library(${CMAKE_PROJECT_NAME}_lib STATIC
         ${SOURCE_FILES}

src/activation_function.hpp

@@ -3,7 +3,6 @@
 
 #include <algorithm>
 #include <cmath>
-#include <numeric>
 #include <vector>
 /**
  * Functor to set the activation function as a Sigmoid function

src/neural_net.cpp

@@ -1,4 +1,5 @@
 #include "neural_net.hpp"
+#include "utility.hpp"
 #include <functional>
 #include <numeric>
 #include <random>
@@ -28,3 +29,37 @@ NeuralNet<ActivationFunction>::NeuralNet(std::vector<size_t> &layer_sizes)
     start_idx += size;
   }
 }
+
+/** Pass input vector through the neural network.
+ * This is a fully connected neural network geometry.
+ * @param x Input vector
+ * @return output of feed forward phase
+ */
+template <class ActivationFunction>
+std::vector<float>
+NeuralNet<ActivationFunction>::feed_forward(std::vector<float> &x) {
+  std::vector<float> A = x;
+  int start_idx = 0;
+
+  // Feed each layer forward except the last layer using the user specified
+  // activation function
+  for (auto size = m_sizes.begin(); size < m_sizes.end() - 1; size++) {
+    // Get the iterator range for the current layer
+    auto layer_start = m_weights.begin() + start_idx;
+    auto layer_end = m_weights.end() + start_idx + *size;
+
+    std::vector<float> Anew = Utilities::feed_layer<ActivationFunction>(
+        layer_start, layer_end, &A, m_activation_func);
+    if (Anew.size() > A.capacity()) {
+      A.reserve(Anew.size());
+    }
+    std::move(Anew.begin(), Anew.end(), A.begin());
+    start_idx += *size;
+  }
+
+  // Always use soft max for the final layer
+  auto last_layer_start = m_weights.begin() + start_idx;
+  auto output = Utilities::feed_layer<SoftMax>(last_layer_start,
+                                               m_weights.end(), A, m_soft_max);
+  return output;
+}

src/neural_net.hpp

@@ -1,6 +1,7 @@
 #ifndef NEURAL_NET_H
 #define NEURAL_NET_H
 
+#include "activation_function.hpp"
 #include <vector>
 template <class ActivationFunction> class NeuralNet {
 public:
@@ -8,8 +9,11 @@ public:
 
 private:
   ActivationFunction m_activation_func;
+  SoftMax m_soft_max;
   std::vector<size_t> m_sizes;
-  std::vector<double> m_weights;
-  std::vector<float> feed_forward(std::vector<float> x);
+  std::vector<float> m_weights;
+  std::vector<float> feed_forward(std::vector<float> &x);
+  std::vector<float> feed_layer_forward(size_t layer_start_idx, size_t size,
+                                        std::vector<float> &A);
 };
 #endif

src/utility.hpp

@@ -0,0 +1,32 @@
+#ifndef UTILITY_H
+#define UTILITY_H
+
+#include <algorithm>
+#include <iterator>
+#include <numeric>
+#include <vector>
+
+namespace Utilities {
+
+template <class ActivationFunction>
+std::vector<float> feed_layer(std::vector<float>::iterator weight_start,
+                              std::vector<float>::iterator weight_end,
+                              std::vector<float> &A,
+                              ActivationFunction activation_func) {
+  // Calculate the new A vector from the current weights
+  std::vector<float> Anew;
+  Anew.reserve(std::distance(weight_start, weight_end));
+  std::transform(weight_start, weight_end, Anew.begin(),
+                 [&A, &activation_func](float weight) {
+                   float summed_weight = std::accumulate(
+                       A.begin(), A.end(), 0.0f, [&weight](float acc, float a) {
+                         return acc + a * weight;
+                       });
+                   return summed_weight;
+                 });
+  activation_func(Anew);
+  return Anew;
+};
+
+} // namespace Utilities
+#endif

tests/unit_tests/CMakeLists.txt

@@ -2,6 +2,7 @@ include_directories(${gtest_SOURCE_DIR}/include ${gtest_SOURCE_DIR})
 
 add_executable(Unit_Tests_run
     test_activation_functions.cpp
+    test_utility.cpp
 )
 
 target_link_libraries(Unit_Tests_run gtest gtest_main)

tests/unit_tests/test_activation_functions.cpp

@@ -1,75 +1,71 @@
-#include <gtest/gtest.h>
 #include "../../src/activation_function.hpp"
 #include <cmath>
+#include <gtest/gtest.h>
 #include <vector>
 
 TEST(ActivationFunctionTest, SigmoidTest) {
-    Sigmoid sigmoid;
-    std::vector<float> input = {0.0, 10.0, -10.0, 1.0, -1.0};
-    std::vector<float> expected = {
-        0.5,
-        0.9999546,
-        0.0000454,
-        1.0 / (1.0 + exp(-1.0)),
-        1.0 / (1.0 + exp(1.0))
-    };
-    
-    std::vector<float> test = input;
-    sigmoid(test);
-    
-    ASSERT_EQ(test.size(), expected.size());
-    for (size_t i = 0; i < test.size(); i++) {
-        EXPECT_NEAR(test[i], expected[i], 1e-6);
-    }
-    
-    // Test initialization standard deviation
-    EXPECT_NEAR(sigmoid.init_stddev(100), sqrt(1.0/100), 1e-6);
+  Sigmoid sigmoid;
+  std::vector<float> input = {0.0, 10.0, -10.0, 1.0, -1.0};
+  std::vector<float> expected = {0.5, 0.9999546, 0.0000454,
+                                 static_cast<float>(1.0 / (1.0 + exp(-1.0))),
+                                 static_cast<float>(1.0 / (1.0 + exp(1.0)))};
+
+  std::vector<float> test = input;
+  sigmoid(test);
+
+  ASSERT_EQ(test.size(), expected.size());
+  for (size_t i = 0; i < test.size(); i++) {
+    EXPECT_NEAR(test[i], expected[i], 1e-6);
+  }
+
+  // Test initialization standard deviation
+  EXPECT_NEAR(sigmoid.init_stddev(100), sqrt(1.0 / 100), 1e-6);
 }
 
 TEST(ActivationFunctionTest, ReLUTest) {
-    ReLU relu;
-    std::vector<float> input = {0.0, 5.0, -5.0, 0.0001, -0.0001};
-    std::vector<float> expected = {0.0, 5.0, 0.0, 0.0001, 0.0};
-    
-    std::vector<float> test = input;
-    relu(test);
-    
-    ASSERT_EQ(test.size(), expected.size());
-    for (size_t i = 0; i < test.size(); i++) {
-        EXPECT_FLOAT_EQ(test[i], expected[i]);
-    }
-    
-    // Test initialization standard deviation
-    EXPECT_NEAR(relu.init_stddev(100), sqrt(2.0/100), 1e-6);
+  ReLU relu;
+  std::vector<float> input = {0.0, 5.0, -5.0, 0.0001, -0.0001};
+  std::vector<float> expected = {0.0, 5.0, 0.0, 0.0001, 0.0};
+
+  std::vector<float> test = input;
+  relu(test);
+
+  ASSERT_EQ(test.size(), expected.size());
+  for (size_t i = 0; i < test.size(); i++) {
+    EXPECT_FLOAT_EQ(test[i], expected[i]);
+  }
+
+  // Test initialization standard deviation
+  EXPECT_NEAR(relu.init_stddev(100), sqrt(2.0 / 100), 1e-6);
 }
 
 TEST(ActivationFunctionTest, SoftMaxTest) {
-    SoftMax softmax;
-    std::vector<float> input = {1.0, 2.0, 3.0, 4.0, 1.0};
-    std::vector<float> test = input;
-    
-    softmax(test);
-    
-    // Test properties of softmax
-    ASSERT_EQ(test.size(), input.size());
-    
-    // Sum should be approximately 1
-    float sum = 0.0;
-    for (float val : test) {
-        sum += val;
-        // All values should be between 0 and 1
-        EXPECT_GE(val, 0.0);
-        EXPECT_LE(val, 1.0);
+  SoftMax softmax;
+  std::vector<float> input = {1.0, 2.0, 3.0, 4.0, 1.0};
+  std::vector<float> test = input;
+
+  softmax(test);
+
+  // Test properties of softmax
+  ASSERT_EQ(test.size(), input.size());
+
+  // Sum should be approximately 1
+  float sum = 0.0;
+  for (float val : test) {
+    sum += val;
+    // All values should be between 0 and 1
+    EXPECT_GE(val, 0.0);
+    EXPECT_LE(val, 1.0);
+  }
+  EXPECT_NEAR(sum, 1.0, 1e-6);
+
+  // Higher input should lead to higher output
+  for (size_t i = 0; i < test.size() - 1; i++) {
+    if (input[i] < input[i + 1]) {
+      EXPECT_LT(test[i], test[i + 1]);
     }
-    EXPECT_NEAR(sum, 1.0, 1e-6);
-    
-    // Higher input should lead to higher output
-    for (size_t i = 0; i < test.size() - 1; i++) {
-        if (input[i] < input[i + 1]) {
-            EXPECT_LT(test[i], test[i + 1]);
-        }
-    }
-    
-    // Test initialization standard deviation
-    EXPECT_NEAR(softmax.init_stddev(100), sqrt(1.0/100), 1e-6);
+  }
+
+  // Test initialization standard deviation
+  EXPECT_NEAR(softmax.init_stddev(100), sqrt(1.0 / 100), 1e-6);
 }

tests/unit_tests/test_utility.cpp

@@ -0,0 +1,80 @@
+#include "activation_function.hpp"
+#include "utility.hpp"
+#include <cmath>
+#include <gtest/gtest.h>
+
+// Simple identity activation function for testing
+struct Identity {
+  void operator()(std::vector<float>& x) const {
+    // Identity function - no change to values
+  }
+};
+
+TEST(UtilityTest, FeedLayerIdentityTest) {
+  // Test with identity activation function for simple verification
+  // Input: [1, 2]
+  // Weights: [0.5, -0.5, 1.0, -1.0]
+  // Expected: [0.5, -1.0] (manually calculated)
+  //   First output: 1.0 * 0.5 + 2.0 * -0.5 = 0.5
+  //   Second output: 1.0 * 1.0 + 2.0 * -1.0 = -1.0
+
+  std::vector<float> weights = {0.5, -0.5, 1.0, -1.0};
+  std::vector<float> input = {1.0, 2.0};
+  Identity identity;
+
+  auto output = Utilities::feed_layer<Identity>(weights.begin(), weights.end(),
+                                               input, identity);
+
+  ASSERT_EQ(output.size(), 2);
+  EXPECT_NEAR(output[0], 0.5f, 1e-5);   // 1.0 * 0.5 + 2.0 * -0.5
+  EXPECT_NEAR(output[1], -1.0f, 1e-5);  // 1.0 * 1.0 + 2.0 * -1.0
+}
+
+TEST(UtilityTest, FeedLayerSigmoidTest) {
+  // Test with sigmoid activation
+  // Input: [1]
+  // Weights: [2, -2]
+  std::vector<float> weights = {2.0, -2.0};
+  std::vector<float> input = {1.0};
+  Sigmoid sigmoid;
+
+  auto output = Utilities::feed_layer<Sigmoid>(weights.begin(), weights.end(),
+                                             input, sigmoid);
+
+  ASSERT_EQ(output.size(), 2);
+  // Note: Sigmoid is applied to the whole vector after matrix multiplication
+  float expected0 = 2.0;  // 1.0 * 2.0
+  float expected1 = -2.0; // 1.0 * -2.0
+  EXPECT_NEAR(output[0], 1.0 / (1.0 + std::exp(-expected0)), 1e-5);
+  EXPECT_NEAR(output[1], 1.0 / (1.0 + std::exp(-expected1)), 1e-5);
+}
+
+TEST(UtilityTest, FeedLayerSoftMaxTest) {
+  // Test with softmax activation
+  // Input: [1]
+  // Weights: [2, 2]
+  std::vector<float> weights = {2.0, 2.0};
+  std::vector<float> input = {1.0};
+  SoftMax softmax;
+
+  auto output = Utilities::feed_layer<SoftMax>(weights.begin(), weights.end(),
+                                             input, softmax);
+
+  ASSERT_EQ(output.size(), 2);
+  // Both outputs should be 0.5 since inputs to softmax are equal (both 2.0)
+  EXPECT_NEAR(output[0], 0.5, 1e-5);
+  EXPECT_NEAR(output[1], 0.5, 1e-5);
+}
+
+TEST(UtilityTest, FeedLayerEmptyInput) {
+  std::vector<float> weights = {1.0, 1.0};
+  std::vector<float> input = {};
+  Identity identity;
+
+  auto output = Utilities::feed_layer<Identity>(weights.begin(), weights.end(),
+                                             input, identity);
+
+  ASSERT_EQ(output.size(), 2);
+  EXPECT_NEAR(output[0], 0.0f, 1e-5);
+  EXPECT_NEAR(output[1], 0.0f, 1e-5);
+}