🎉 Network Wrapper completely rewritten and tested

Author: JyotinderSingh

README.md

@@ -8,28 +8,30 @@ The framework while being extremely easy to use also allows a relatively high de
 All implementations are fully documented to help understand the underlying logic for each component better.
 
 
+#### The following files contain example code for using the Network wrapper.
+- [demo_Network_categorical.py](./demo_Network_categorical.py)
+- [demo_Network_binary_logistic_regression.py](./demo_Network_binary_logistic_regression.py)
+- [demo_Network_linear_regression.py](./demo_Network_linear_regression.py)
 
 #### The following files contain example code to get you started.
-- [demo_multiclass_classification.py](./demo_multiclass_classification.py) 
+- [demo_categorical.py](./demo_categorical.py) 
 - [demo_binary_logistic_regression.py](./demo_binary_logistic_regression.py)
+- [demo_linear_regression.py](./demo_linear_regression.py)
 
-#### The [NetworkDemo.py](NetworkDemo.py) file contains example code for using the Network wrapper.
-
-## *Features*
-- Fully connected Layers
-- Dropout Layers
-- ReLU, Softmax, Sigmoid activations
-- Generate training data (Classification data)
-- Categorical Cross Entropy Loss, Binary Cross Entropy Loss
+## **Features**
+- Fully connected, Dropout Layers
+- ReLU, Softmax, Sigmoid, and Linear activations
+- Generate synthetic training data (Classification, and Regression data)
+- Categorical Cross Entropy Loss, Binary Cross Entropy Loss, Mean Squared Error
 - L1 & L2 regularization
 - Backpropogation
 - Optimizers
   - SGD (with decay and momentum)
   - AdaGrad
   - RMSprop
   - Adam
-- Network Wrapper with TinyFlow backend (under development)
-  - The wrapper currently supports setting up a basic Neural Network and running a simple training loop using the above components, without having to worry about instantiating and getting the dimensions right for every layer.
+- Network Wrapper with TinyFlow backend
+  - The wrapper supports setting up a Neural Network and running a simple training loop using the above components, without having to write long error prone code.
 
 ## Steps to use
 ```

TinyFlow/Activations.py

@@ -7,7 +7,7 @@ class Activation_ReLU:
     max (0, input)
     '''
 
-    def forward(self, inputs):
+    def forward(self, inputs, training):
         '''Activation_ReLU.forward (input_data)'''
         # np.maximum takes two inputs and finds element wise maximum
         self.output = np.maximum(0, inputs)
@@ -39,7 +39,7 @@ class Activation_Softmax:
     '''Softmax activation'''
 
     # Forward Pass
-    def forward(self, inputs):
+    def forward(self, inputs, training):
         '''Activation_Softmax.forward (input_data)'''
 
         # get unnormalized probabilities
@@ -71,7 +71,7 @@ def __str__(self):
 # Sigmoid activation
 class Activation_Sigmoid:
     # Forward Pass
-    def forward(self, inputs):
+    def forward(self, inputs, training):
         # Save input and calcuilate/save output of the sigmoid function
         self.input = inputs
         self.output = 1 / (1 + np.exp(-inputs))
@@ -93,7 +93,7 @@ def __str__(self):
 class Activation_Linear:
 
     # Forward pass
-    def forward(self, inputs):
+    def forward(self, inputs, training):
         # All you need to do is just cache the values
         self.input = inputs
         self.output = inputs

TinyFlow/Layers.py

@@ -19,7 +19,7 @@ def __init__(self, inputs, neurons, weight_regularizer_l1=0, weight_regularizer_
         self.bias_regularizer_l2 = bias_regulariser_l2
 
     # Forward Pass
-    def forward(self, inputs):
+    def forward(self, inputs, training):
         '''Layer_Dense.forward (input_data)'''
 
         # Calculate the output values from inputs, weights, and biases
@@ -78,10 +78,16 @@ def __init__(self, rate):
         self.rate = 1 - rate
 
     # Forward pass
-    def forward(self, values):
+    def forward(self, values, training=True):
         # save input values
         self.input = values
 
+        # If not in training mode - return values
+        if not training:
+            self.output = values.copy()
+            return
+
+        # Generate and save scaled mask
         self.binary_mask = np.random.binomial(
             1, self.rate, size=values.shape) / self.rate
 

TinyFlow/Loss.py

@@ -86,7 +86,7 @@ def remember_trainable_layers(self, trainable_layers):
     # Calculates the data and regularization losses
     # given model output and ground truth values
 
-    def calculate(self, output, y):
+    def calculate(self, output, y, *, include_regularization=False):
         '''calculate(self, output, ground_truth)\n
             internal method for Network wrapper\n
             Calculates the data and regularization losses
@@ -99,6 +99,10 @@ def calculate(self, output, y):
         # Calculate the mean loss
         data_loss = np.mean(sample_losses)
 
+        # If just data loss is needed, return it
+        if not include_regularization:
+            return data_loss
+
         # Return the data and regularization losses
         return data_loss, self.network_regularization_loss()
 

TinyFlow/Metrics.py

@@ -38,6 +38,27 @@ def compare(self, predictions, y):
         return np.absolute(predictions - y) < self.precision
 
 
+class Accuracy_Categorical(Accuracy):
+
+    # No initalization is needed
+    def init(self, y):
+        pass
+
+    # Compares predictions to the ground truth values
+    def compare(self, predictions, y):
+        return predictions == y
+
+
+# class Accuracy_BinaryLogisticRegression(Accuracy):
+
+#     # No initialization is needed
+#     def init(self, y):
+#         pass
+
+#     def compare(self, predictions, y):
+#         return predictions == y
+
+
 def model_accuracy_softmax(outputs, labels):
     '''Returns the accuracy of the model on the current batch'''
 

TinyFlow/Network.py

@@ -1,7 +1,3 @@
-######################
-## UNDER DEVELOPMENT##
-######################
-
 # Network class
 class Network:
 
@@ -65,12 +61,12 @@ def forward(self, X, training):
         # Call the forward method on the input layer
         # This will set the output property that the first layer
         # is expecting as the 'prev' object
-        self.input_layer.forward(X)
+        self.input_layer.forward(X, training)
 
         # Call the forward methid of every object in sequence
         # while passing the output property of the previous object as a parameter
         for layer in self.layers:
-            layer.forward(layer.prev.output)
+            layer.forward(layer.prev.output, training)
 
         # 'layer' is now the last object from the list, return its output
         return layer.output
@@ -99,7 +95,7 @@ def train(self, X, y, *, epochs=1, print_every=1, validation_data=None):
             output = self.forward(X, training=True)
 
             # Calculate loss
-            data_loss, regularization_loss = self.loss.calculate(output, y)
+            data_loss, regularization_loss = self.loss.calculate(output, y, include_regularization=True)
 
             loss = data_loss + regularization_loss
 
@@ -118,11 +114,29 @@ def train(self, X, y, *, epochs=1, print_every=1, validation_data=None):
 
             # Print a summary
             if not epoch % print_every:
-                print(f'epoch: {epoch}, acc: {accuracy:.3f}, loss: {loss:.3f} (data_loss: {data_loss:.3f}, reg_loss: {regularization_loss:.3f}), lr: {self.optimizer.current_learning_rate}')
+                print(f'\nepoch: {epoch}\nacc: {accuracy:.3f}, loss: {loss:.3f} (data_loss: {data_loss:.3f}, reg_loss: {regularization_loss:.3f}), lr: {self.optimizer.current_learning_rate}')
+
+        # If validation data has been provided
+        if validation_data is not None:
+
+            # For better readability
+            X_val, y_val = validation_data
+
+            # Perform forward pass
+            output = self.forward(X_val, training=False)
+
+            # Calculate the loss
+            loss = self.loss.calculate(output, y_val)
+
+            # Get predictions and calculate validation accuracy
+            predictions = self.output_layer_activation.predictions(output)
+            accuracy = self.accuracy.calculate(predictions, y_val)
 
+            # Print a sumarry
+            print(f'\nvalidation,\nacc: {accuracy:.3f}, loss: {loss:.3f}')
 
 class Layer_Input:
 
     # Pass the input
-    def forward(self, inputs):
+    def forward(self, inputs, training):
         self.output = inputs

TinyFlow/Network_DEPRECATED.py

@@ -1,3 +1,10 @@
+############################################################################
+#***************************************************************************
+# THIS VERSION OF THE NETWORK WRAPPER HAS BEEN DEPRECATED
+# It is HIGHLY RECOMMENDED to use the more stable and flexible Network class
+#***************************************************************************
+############################################################################
+
 import numpy as np
 from TinyFlow import Layers
 from TinyFlow import Activations

demo_Network_DEPRECATED.py

@@ -1,3 +1,10 @@
+############################################################################
+#***************************************************************************
+# THIS FILE USES A DEPRECATED VERSION OF THE NETWORK WRAPPER
+# It is HIGHLY RECOMMENDED to use the more stable and flexible Network class
+#***************************************************************************
+############################################################################
+
 from TinyFlow.Network_DEPRECATED import Network
 from TinyFlow.Loss import Loss_CategoricalCrossEntropy
 from TinyFlow.Optimizers import Optimizer_Adam

demo_Network_binary_logistic_regression.py

@@ -0,0 +1,39 @@
+# Demo for Network wrapper to implement Binary Logistic Regression
+import numpy as np
+from TinyFlow.Network import Network
+from TinyFlow.Datasets import spiral_data
+from TinyFlow.Layers import Layer_Dense
+from TinyFlow.Activations import Activation_Sigmoid, Activation_ReLU
+from TinyFlow.Optimizers import Optimizer_Adam
+from TinyFlow.Loss import Loss_BinaryCrossEntropy
+from TinyFlow.Metrics import Accuracy_Categorical
+
+# Create train and test set
+X, y = spiral_data(100, 2)
+X_test, y_test = spiral_data(100, 2)
+
+# Reshape labels to be a list of lists
+# Inner list contains one output (either 0 or 1)
+# per each output neuron, 1 in this case
+y = y.reshape(-1, 1)
+y_test = y_test.reshape(-1, 1)
+
+# Instantiate the model
+model = Network()
+
+# Add layers
+model.add(Layer_Dense(2, 64, weight_regularizer_l2=5e-4, bias_regulariser_l2=5e-4))
+model.add(Activation_ReLU())
+model.add(Layer_Dense(64, 1))
+model.add(Activation_Sigmoid())
+
+# Set loss, optimizer, and accuracy
+model.set(loss=Loss_BinaryCrossEntropy(), optimizer=Optimizer_Adam(
+    decay=1e-8), accuracy=Accuracy_Categorical())
+
+# Compile the model
+model.compile_model()
+
+# Train the model
+model.train(X, y, epochs=10000, print_every=100,
+            validation_data=(X_test, y_test))

demo_Network_categorical.py

@@ -0,0 +1,33 @@
+# Demo for Network wrapper to implement Multiclass Classification
+import numpy as np
+from TinyFlow.Network import Network
+from TinyFlow.Datasets import spiral_data
+from TinyFlow.Layers import Layer_Dense, Layer_Dropout
+from TinyFlow.Activations import Activation_Softmax, Activation_ReLU
+from TinyFlow.Optimizers import Optimizer_Adam
+from TinyFlow.Loss import Loss_CategoricalCrossEntropy
+from TinyFlow.Metrics import Accuracy_Categorical
+
+# Create dataset
+X, y = spiral_data(1000, 3)
+X_test, y_test = spiral_data(100, 3)
+
+# Instantiate the model
+model = Network()
+
+# Add layers
+model.add(Layer_Dense(2, 512, weight_regularizer_l2=5e-4, bias_regulariser_l2=5e-4))
+model.add(Activation_ReLU())
+model.add(Layer_Dropout(0.1))
+model.add(Layer_Dense(512, 3))
+model.add(Activation_Softmax())
+
+# Set loss, optimizer, and accuracy objects
+model.set(loss=Loss_CategoricalCrossEntropy(), optimizer=Optimizer_Adam(
+    learning_rate=0.05, decay=1e-8), accuracy=Accuracy_Categorical())
+
+# Compile the model
+model.compile_model()
+
+# Train the model
+model.train(X, y, epochs=10000, print_every=100, validation_data=(X_test, y_test))