Skip to content

Commit 39cb8a0

Browse files
SkibidiProductionSkibidiProduction
authored
Merge pull request #375 from apphp/SAM-8-
Sam 8 LeakyReLU and refactoring
2 parents 2872f1f + 6c4d9f4 commit 39cb8a0

File tree

18 files changed

+409
-163
lines changed

18 files changed

+409
-163
lines changed

‎docs/images/activation-functions/hard-silu-derivative.png

25.7 KB
Loading

‎docs/images/activation-functions/hard-silu.png

14.5 KB
Loading

‎src/NeuralNet/ActivationFunctions/Base/Contracts/IBufferDerivative.php

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,15 @@
1313
* @package Rubix/ML
1414
* @author Andrew DalPino
1515
* @author Aleksei Nechaev <omfg.rus@gmail.com>
16+
* @author Samuel Akopyan <leumas.a@gmail.com>
1617
*/
1718
interface IBufferDerivative extends Derivative
1819
{
1920
/**
2021
* Calculate the derivative of the single parameter.
2122
*
22-
* @param NDArray $x Input matrix
23+
* @param NDArray $input Input matrix
2324
* @return NDArray Derivative matrix
2425
*/
25-
public function differentiate(NDArray $x) : NDArray;
26+
public function differentiate(NDArray $input) : NDArray;
2627
}

‎src/NeuralNet/ActivationFunctions/Base/Contracts/IOBufferDerivative.php

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,25 @@
22

33
namespace Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts;
44

5+
use NDArray;
6+
7+
/**
8+
* Derivative based on input / output buffer
9+
*
10+
* @category Machine Learning
11+
* @package Rubix/ML
12+
* @author Andrew DalPino
13+
* @author Aleksei Nechaev <omfg.rus@gmail.com>
14+
* @author Samuel Akopyan <leumas.a@gmail.com>
15+
*/
516
interface IOBufferDerivative
617
{
18+
/**
19+
* Calculate the derivative of the activation.
20+
*
21+
* @param NDArray $input Input matrix
22+
* @param NDArray $output Output matrix
23+
* @return NDArray Derivative matrix
24+
*/
25+
public function differentiate(NDArray $input, NDArray $output) : NDArray;
726
}

‎src/NeuralNet/ActivationFunctions/Base/Contracts/OBufferDerivative.php

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,24 @@
22

33
namespace Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts;
44

5+
use NDArray;
6+
7+
/**
8+
* Derivative based on output buffer
9+
*
10+
* @category Machine Learning
11+
* @package Rubix/ML
12+
* @author Andrew DalPino
13+
* @author Aleksei Nechaev <omfg.rus@gmail.com>
14+
* @author Samuel Akopyan <leumas.a@gmail.com>
15+
*/
516
interface OBufferDerivative
617
{
18+
/**
19+
* Calculate the derivative of the activation.
20+
*
21+
* @param NDArray $output Output matrix
22+
* @return NDArray Derivative matrix
23+
*/
24+
public function differentiate(NDArray $output) : NDArray;
725
}

‎src/NeuralNet/ActivationFunctions/ELU/ELU.php

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
use NumPower;
88
use NDArray;
99
use Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts\ActivationFunction;
10-
use Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts\IBufferDerivative;
10+
use Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts\IOBufferDerivative;
1111
use Rubix\ML\NeuralNet\ActivationFunctions\ELU\Exceptions\InvalidAlphaException;
1212

1313
/**
@@ -26,7 +26,7 @@
2626
* @author Aleksei Nechaev <omfg.rus@gmail.com>
2727
* @author Samuel Akopyan <leumas.a@gmail.com>
2828
*/
29-
class ELU implements ActivationFunction, IBufferDerivative
29+
class ELU implements ActivationFunction, IOBufferDerivative
3030
{
3131
/**
3232
* Class constructor.
@@ -40,7 +40,7 @@ public function __construct(protected float $alpha = 1.0)
4040
{
4141
if ($this->alpha < 0.0) {
4242
throw new InvalidAlphaException(
43-
message: "Alpha must be greater than 0, $alpha given."
43+
message: "Alpha must be greater than 0, {$this->alpha} given."
4444
);
4545
}
4646
}
@@ -71,28 +71,29 @@ public function activate(NDArray $input) : NDArray
7171
}
7272

7373
/**
74-
* Calculate the derivative of the activation function.
74+
* Calculate the derivative of the ELU activation function using input and output.
7575
*
76-
* f'(x) = 1 if x > 0
77-
* f'(x) = α * e^x if x ≤ 0
76+
* f'(x) = 1 if x > 0
77+
* f'(x) = f(x) + α if x ≤ 0, where f(x) is the ELU output
7878
*
79-
* @param NDArray $x Output matrix
80-
* @return NDArray Derivative matrix
79+
* @param NDArray $input Input matrix (used to determine x > 0 mask)
80+
* @param NDArray $output Output from the ELU activation function
81+
* @return NDArray Derivative matrix
8182
*/
82-
public function differentiate(NDArray $x) : NDArray
83+
public function differentiate(NDArray $input, NDArray $output): NDArray
8384
{
8485
// For x > 0: 1
85-
$positivePart = NumPower::greater($x, 0);
86+
$positiveMask = NumPower::greater($input, 0);
8687

87-
// For x <= 0: α * e^x
88-
$negativeMask = NumPower::lessEqual($x, 0);
88+
// For x <= 0: output + α
89+
$negativeMask = NumPower::lessEqual($input, 0);
8990
$negativePart = NumPower::multiply(
90-
NumPower::multiply($negativeMask, NumPower::exp($x)),
91-
$this->alpha
91+
NumPower::add($output, $this->alpha),
92+
$negativeMask
9293
);
9394

9495
// Combine both parts
95-
return NumPower::add($positivePart, $negativePart);
96+
return NumPower::add($positiveMask, $negativePart);
9697
}
9798

9899
/**

‎src/NeuralNet/ActivationFunctions/GELU/GELU.php

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -87,19 +87,19 @@ public function activate(NDArray $input) : NDArray
8787
* - β = 0.044715
8888
* - sech^2(z) = (1/cosh(z))^2
8989
*
90-
* @param NDArray $x Output matrix
90+
* @param NDArray $input Input matrix
9191
* @return NDArray Derivative matrix
9292
*/
93-
public function differentiate(NDArray $x) : NDArray
93+
public function differentiate(NDArray $input) : NDArray
9494
{
9595
// Calculate x^3
96-
$cubed = NumPower::pow($x, 3);
96+
$cubed = NumPower::pow($input, 3);
9797

9898
// Calculate inner term: ALPHA * (x + BETA * x^3)
9999
$innerTerm = NumPower::multiply(
100100
self::ALPHA,
101101
NumPower::add(
102-
$x,
102+
$input,
103103
NumPower::multiply(self::BETA, $cubed)
104104
)
105105
);
@@ -122,15 +122,15 @@ public function differentiate(NDArray $x) : NDArray
122122
NumPower::multiply(
123123
NumPower::multiply(
124124
0.5 * self::ALPHA,
125-
$x
125+
$input
126126
),
127127
$sech2
128128
),
129129
NumPower::add(
130130
1.0,
131131
NumPower::multiply(
132132
3.0 * self::BETA,
133-
NumPower::pow($x, 2)
133+
NumPower::pow($input, 2)
134134
)
135135
)
136136
);

‎src/NeuralNet/ActivationFunctions/HardSiLU/HardSiLU.php

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -63,19 +63,19 @@ public function activate(NDArray $input) : NDArray
6363
*
6464
* f'(x) = HardSigmoid(x) + x * HardSigmoid'(x)
6565
*
66-
* @param NDArray $x Input matrix
66+
* @param NDArray $input Input matrix
6767
* @return NDArray Derivative matrix
6868
*/
69-
public function differentiate(NDArray $x) : NDArray
69+
public function differentiate(NDArray $input) : NDArray
7070
{
7171
// Calculate HardSigmoid(x)
72-
$hardSigmoid = $this->hardSigmoid->activate($x);
72+
$hardSigmoid = $this->hardSigmoid->activate($input);
7373

7474
// Calculate HardSigmoid'(x)
75-
$hardSigmoidDerivative = $this->hardSigmoid->differentiate($x);
75+
$hardSigmoidDerivative = $this->hardSigmoid->differentiate($input);
7676

7777
// Calculate x * HardSigmoid'(x)
78-
$xTimesDerivative = NumPower::multiply($x, $hardSigmoidDerivative);
78+
$xTimesDerivative = NumPower::multiply($input, $hardSigmoidDerivative);
7979

8080
// Calculate HardSigmoid(x) + x * HardSigmoid'(x)
8181
return NumPower::add($hardSigmoid, $xTimesDerivative);

‎src/NeuralNet/ActivationFunctions/HardSigmoid/HardSigmoid.php

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -76,17 +76,17 @@ public function activate(NDArray $input) : NDArray
7676
/**
7777
* Calculate the derivative of the activation function.
7878
*
79-
* f'(x) = 0.2 if -2.5 < x < 2.5
79+
* f'(x) = 0.2 if -2.5 <= x <= 2.5
8080
* f'(x) = 0 otherwise
8181
*
82-
* @param NDArray $x Input matrix
82+
* @param NDArray $input Input matrix
8383
* @return NDArray Derivative matrix
8484
*/
85-
public function differentiate(NDArray $x) : NDArray
85+
public function differentiate(NDArray $input) : NDArray
8686
{
87-
// For values in the linear region (-2.5 < x < 2.5): SLOPE
88-
$inLinearRegion = NumPower::greater($x, self::LOWER_BOUND);
89-
$inLinearRegion = NumPower::multiply($inLinearRegion, NumPower::less($x, self::UPPER_BOUND));
87+
// For values in the linear region (-2.5 <= x <= 2.5): SLOPE
88+
$inLinearRegion = NumPower::greaterEqual($input, self::LOWER_BOUND);
89+
$inLinearRegion = NumPower::multiply($inLinearRegion, NumPower::lessEqual($input, self::UPPER_BOUND));
9090
$linearPart = NumPower::multiply($inLinearRegion, self::SLOPE);
9191

9292
// For values outside the linear region: 0

‎src/NeuralNet/ActivationFunctions/HyperbolicTangent/HyperbolicTangent.php

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
use NumPower;
88
use NDArray;
99
use Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts\ActivationFunction;
10-
use Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts\IBufferDerivative;
10+
use Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts\OBufferDerivative;
1111

1212
/**
1313
* Hyperbolic Tangent
@@ -20,7 +20,7 @@
2020
* @author Andrew DalPino
2121
* @author Samuel Akopyan <leumas.a@gmail.com>
2222
*/
23-
class HyperbolicTangent implements ActivationFunction, IBufferDerivative
23+
class HyperbolicTangent implements ActivationFunction, OBufferDerivative
2424
{
2525
/**
2626
* Apply the Hyperbolic Tangent activation function to the input.

0 commit comments

Comments
 (0)