Creating a Simple Neural Network

# SINGLE LAYER WITH NUMPY
import numpy as np

# Single sample data (consistent with previous examples)
inputs = [2.1, 1.7, 3.3, 0.8]

# Weights for 3 neurons, each with 4 inputs
weights = [[0.5, -0.2, 0.9, -0.3],
           [0.7, 0.4, -0.6, 0.8],
           [-0.1, 0.6, 0.2, -0.7]]

biases = [1.5, 2.2, 0.9]

# Convert to NumPy arrays
inputs_array = np.array(inputs)
weights_array = np.array(weights)
biases_array = np.array(biases)

# Calculate outputs using matrix multiplication
outputs = np.dot(weights_array, inputs_array) + biases_array

print(outputs)

# 2 LAYERS AND BATCH OF DATA USING NUMPY
import numpy as np

# Single input data (consistent with previous examples)
inputs = [2.1, 1.7, 3.3, 0.8]

# First layer weights and biases
weights = [[0.5, -0.2, 0.9, -0.3],
           [0.7, 0.4, -0.6, 0.8],
           [-0.1, 0.6, 0.2, -0.7]]
biases = [1.5, 2.2, 0.9]

# Second layer weights and biases
weights2 = [[0.3, -0.4, 0.6],
            [0.2, 0.5, -0.3],
            [0.8, -0.1, 0.4]]
biases2 = [0.7, 1.2, -0.5]

# Convert to NumPy arrays
inputs_array = np.array(inputs)
weights_array = np.array(weights)
biases_array = np.array(biases)
weights2_array = np.array(weights2)
biases2_array = np.array(biases2)

# First layer forward pass
layer1_outputs = np.dot(weights_array, inputs_array) + biases_array

# Second layer forward pass
layer2_outputs = np.dot(weights2_array, layer1_outputs) + biases2_array

print("Layer 1 outputs:")
print(layer1_outputs)
print("\nLayer 2 outputs:")
print(layer2_outputs)

# ACTIVATION FUNCTION: RELU
import numpy as np

# Basic ReLU demonstration
inputs = [0, 2, -1, 3.3, -2.7, 1.1, 2.2, -100]
output = np.maximum(0, inputs)
print(output)

# ReLU activation class
class Activation_ReLU:
    def forward(self, inputs):
        self.output = np.maximum(0, inputs)

# Using our previous data
X, y = spiral_data(samples=100, classes=5)
dense1 = Layer_Dense(2, 5)
activation1 = Activation_ReLU()

# Forward pass through dense layer
dense1.forward(X)
# Forward pass through ReLU activation
activation1.forward(dense1.output)

print(activation1.output[:5])

# ACTIVATION FUNCTION: SOFTMAX
import numpy as np

# Manual Softmax calculation
inputs = [[1, 2, 3, 2.5],
          [2, 5, -1, 2],
          [-1.5, 2.7, 3.3, -0.8]]

# Get unnormalized probabilities
exp_values = np.exp(inputs - np.max(inputs, axis=1, keepdims=True))
# Normalize for each sample
probabilities = exp_values / np.sum(exp_values, axis=1, keepdims=True)
print(probabilities)
print(np.sum(probabilities, axis=1))

# Softmax activation class
class Activation_Softmax:
    def forward(self, inputs):
        # Get unnormalized probabilities
        exp_values = np.exp(inputs - np.max(inputs, axis=1, keepdims=True))
        # Normalize for each sample
        probabilities = exp_values / np.sum(exp_values, axis=1, keepdims=True)
        self.output = probabilities

# CROSS-ENTROPY LOSS IMPLEMENTATION
import numpy as np

# Cross-entropy loss class
class Loss_CategoricalCrossentropy:
    def forward(self, y_pred, y_true):
        # Number of samples in a batch
        samples = len(y_pred)
        # Clip data to prevent division by 0
        y_pred_clipped = np.clip(y_pred, 1e-7, 1 - 1e-7)
        
        # Probabilities for target values - only if categorical labels
        if len(y_true.shape) == 1:
            correct_confidences = y_pred_clipped[
                range(samples),
                y_true
            ]
        # Mask values - only for one-hot encoded labels
        elif len(y_true.shape) == 2:
            correct_confidences = np.sum(
                y_pred_clipped * y_true,
                axis=1
            )
        
        # Losses
        negative_log_likelihoods = -np.log(correct_confidences)
        return negative_log_likelihoods

# Example usage
softmax_outputs = np.array([[0.7, 0.1, 0.2],
                            [0.1, 0.5, 0.4],
                            [0.02, 0.0, 0.98]])
class_targets = np.array([[1, 0, 0],
                          [0, 1, 0],
                          [0, 0, 1]])

loss_function = Loss_CategoricalCrossentropy()
loss = loss_function.calculate(softmax_outputs, class_targets)
print('loss:', loss)

# COMPLETE NEURAL NETWORK FORWARD PASS
import numpy as np

# Create dataset
X, y = spiral_data(samples=100, classes=5)

# Create first Dense layer with 2 input features and 5 output values
dense1 = Layer_Dense(2, 5)
# Create ReLU activation
activation1 = Activation_ReLU()

# Create second Dense layer with 5 input features and 5 output values
dense2 = Layer_Dense(5, 5)
# Create Softmax activation
activation2 = Activation_Softmax()

# Create loss function
loss_function = Loss_CategoricalCrossentropy()

# Forward pass through first layer
dense1.forward(X)
activation1.forward(dense1.output)

# Forward pass through second layer
dense2.forward(activation1.output)
activation2.forward(dense2.output)

# Calculate loss and accuracy
loss = loss_function.calculate(activation2.output, y)
predictions = np.argmax(activation2.output, axis=1)
if len(y.shape) == 2:
    y = np.argmax(y, axis=1)
accuracy = np.mean(predictions == y)

print('First 5 predictions:')
print(activation2.output[:5])
print('loss:', loss)
print('accuracy:', accuracy)