Skip to content

Activation functions

BaseActivation

Bases: BaseLayer

Template for activation layers.

Activation functions do not need to update or expose their weights, since they do not have trainable parameters

Source code in src/nnfs/activations.py 
 6
 7
 8
 9
10
11
12
13
14
15
class BaseActivation(BaseLayer):
    """Template for activation layers.

    Activation functions do not need to update or expose their weights,
    since they do not have trainable parameters
    """

    @property
    def trainable(self):
        return None

Sigmoid

Bases: BaseActivation

Sigmoid activation layer.

Applies the sigmoid function element-wise to the input during forward propagation, and computes its derivative during backpropagation.

Attributes:

Name Type Description
output ndarray

Stores the output from the forward pass for use in the backward pass.
layer_name str

Short name for the layer type.
index int

Position of the layer within the full model (initializes at 0).
Source code in src/nnfs/activations.py 
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
class Sigmoid(BaseActivation):
    """Sigmoid activation layer.

    Applies the sigmoid function element-wise to the input during forward propagation,
    and computes its derivative during backpropagation.

    Attributes:
        output (np.ndarray): Stores the output from the forward pass for use in the backward pass.
        layer_name (str): Short name for the layer type.
        index (int): Position of the layer within the full model (initializes at 0).
    """

    def __init__(self):
        self.output: np.ndarray = np.zeros(1)

        # attributes for layer navigation
        self.layer_name: str = "Sigmoid"
        self.index: int = 0

    def forward(self, X_input: np.ndarray) -> np.ndarray:
        """Performs the forward pass using the sigmoid activation.

        Args:
            X_input (np.ndarray): Input array to the layer.

        Returns:
            Activated output after applying sigmoid.
        """
        output = func_sigmoid(X_input)
        self.output = output
        return output

    def backward(self, grad_next: np.ndarray) -> np.ndarray:
        """Performs the backward pass for the sigmoid layer.

        Computes the gradient of the loss with respect to the input,
        reusing the cached output from the forward pass.

        Args:
            grad_next (np.ndarray): Gradient from the next layer.

        Returns:
            Gradient of the loss with respect to this layer's input.
        """
        return grad_next * self.output * (1 - self.output)

forward(X_input)

Performs the forward pass using the sigmoid activation.

Parameters:

Name Type Description Default
X_input ndarray

Input array to the layer.

 required

Returns:

Type Description
ndarray

Activated output after applying sigmoid.
Source code in src/nnfs/activations.py 
41
42
43
44
45
46
47
48
49
50
51
52
def forward(self, X_input: np.ndarray) -> np.ndarray:
    """Performs the forward pass using the sigmoid activation.

    Args:
        X_input (np.ndarray): Input array to the layer.

    Returns:
        Activated output after applying sigmoid.
    """
    output = func_sigmoid(X_input)
    self.output = output
    return output

backward(grad_next)

Performs the backward pass for the sigmoid layer.

Computes the gradient of the loss with respect to the input, reusing the cached output from the forward pass.

Parameters:

Name Type Description Default
grad_next ndarray

Gradient from the next layer.

 required

Returns:

Type Description
ndarray

Gradient of the loss with respect to this layer's input.
Source code in src/nnfs/activations.py 
54
55
56
57
58
59
60
61
62
63
64
65
66
def backward(self, grad_next: np.ndarray) -> np.ndarray:
    """Performs the backward pass for the sigmoid layer.

    Computes the gradient of the loss with respect to the input,
    reusing the cached output from the forward pass.

    Args:
        grad_next (np.ndarray): Gradient from the next layer.

    Returns:
        Gradient of the loss with respect to this layer's input.
    """
    return grad_next * self.output * (1 - self.output)

Softmax

Bases: BaseActivation

Softmax activation layer.

Applies the softmax function the input during forward propagation, and computes its derivative during backpropagation.

Attributes:

Name Type Description
output ndarray

Stores the output from the forward pass for use in the backward pass.
layer_name str

Short name for the layer type.
index int

Position of the layer within the full model (initializes at 0).
Source code in src/nnfs/activations.py 
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
class Softmax(BaseActivation):
    """Softmax activation layer.

    Applies the softmax function the input during forward propagation,
    and computes its derivative during backpropagation.

    Attributes:
        output (np.ndarray): Stores the output from the forward pass for use in the backward pass.
        layer_name (str): Short name for the layer type.
        index (int): Position of the layer within the full model (initializes at 0).
    """

    def __init__(self):
        self.output: np.ndarray = np.zeros(1)

        # attributes for layer navigation
        self.layer_name: str = "Softmax"
        self.index: int = 0

    def forward(self, X_input: np.ndarray) -> np.ndarray:
        """Performs the forward pass using the softmax activation.

        Args:
            X_input (np.ndarray): Input array to the layer.

        Returns:
            Activated output after applying softmax.
        """
        output = func_softmax(X_input)
        self.output = output
        return output

    def backward(self, grad_next: np.ndarray) -> np.ndarray:
        """Performs the backward pass for the softmax layer.

        Computes the gradient of the loss with respect to the input,
        reusing the cached output from the forward pass.

        Args:
            grad_next (np.ndarray): Gradient from the next layer.

        Returns:
            Gradient of the loss with respect to this layer's input.
        """
        # softmax probabilities
        s = self.output  # shape (batch_size, num_classes)

        # compute the dot product of s * grad_next for each sample
        s_dot_grad = np.sum(s * grad_next, axis=1, keepdims=True)

        # Jacobian-vector product: dx = s * (grad_next - s_dot_grad)
        dx = s * (grad_next - s_dot_grad)
        return dx

forward(X_input)

Performs the forward pass using the softmax activation.

Parameters:

Name Type Description Default
X_input ndarray

Input array to the layer.

 required

Returns:

Type Description
ndarray

Activated output after applying softmax.
Source code in src/nnfs/activations.py 
 99
100
101
102
103
104
105
106
107
108
109
110
def forward(self, X_input: np.ndarray) -> np.ndarray:
    """Performs the forward pass using the softmax activation.

    Args:
        X_input (np.ndarray): Input array to the layer.

    Returns:
        Activated output after applying softmax.
    """
    output = func_softmax(X_input)
    self.output = output
    return output

backward(grad_next)

Performs the backward pass for the softmax layer.

Computes the gradient of the loss with respect to the input, reusing the cached output from the forward pass.

Parameters:

Name Type Description Default
grad_next ndarray

Gradient from the next layer.

 required

Returns:

Type Description
ndarray

Gradient of the loss with respect to this layer's input.
Source code in src/nnfs/activations.py 
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
def backward(self, grad_next: np.ndarray) -> np.ndarray:
    """Performs the backward pass for the softmax layer.

    Computes the gradient of the loss with respect to the input,
    reusing the cached output from the forward pass.

    Args:
        grad_next (np.ndarray): Gradient from the next layer.

    Returns:
        Gradient of the loss with respect to this layer's input.
    """
    # softmax probabilities
    s = self.output  # shape (batch_size, num_classes)

    # compute the dot product of s * grad_next for each sample
    s_dot_grad = np.sum(s * grad_next, axis=1, keepdims=True)

    # Jacobian-vector product: dx = s * (grad_next - s_dot_grad)
    dx = s * (grad_next - s_dot_grad)
    return dx