Skip to content

Add Rectified Linear Unit and Leaky Rectified Linear Unit activation functions #74

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

Add Rectified Linear Unit and Leaky Rectified Linear Unit activation functions #74

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
588151b
Added ReLU and LReLU functionality
Jun 24, 2016
14f7b01
fix indentation issues
Aug 20, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions src/fann.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -769,6 +769,12 @@ FANN_EXTERNAL fann_type *FANN_API fann_run(struct fann * ann, fann_type * input)
case FANN_LINEAR_PIECE_SYMMETRIC:
neuron_it->value = (fann_type)((neuron_sum < -multiplier) ? -multiplier : (neuron_sum > multiplier) ? multiplier : neuron_sum);
break;
case FANN_RELU:
neuron_it->value = (fann_type)(neuron_sum > 0 ? neuron_sum : 0);
break;
case FANN_LEAKY_RELU:
neuron_it->value = (fann_type)(neuron_sum > 0 ? neuron_sum : neuron_sum / 100);
break;
case FANN_ELLIOT:
case FANN_ELLIOT_SYMMETRIC:
case FANN_GAUSSIAN:
Expand Down
6 changes: 6 additions & 0 deletions src/fann_train.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@ fann_type fann_activation_derived(unsigned int activation_function,
return (fann_type) fann_cos_derive(steepness, sum);
case FANN_THRESHOLD:
fann_error(NULL, FANN_E_CANT_TRAIN_ACTIVATION);
case FANN_RELU:
return (fann_type) fann_relu_derive(steepness, sum);
case FANN_LEAKY_RELU:
return (fann_type) fann_leaky_relu_derive(steepness, sum);
}
return 0;
}
Expand Down Expand Up @@ -133,6 +137,8 @@ fann_type fann_update_MSE(struct fann *ann, struct fann_neuron* neuron, fann_typ
case FANN_LINEAR_PIECE:
case FANN_SIN:
case FANN_COS:
case FANN_RELU:
case FANN_LEAKY_RELU:
break;
}

Expand Down
2 changes: 1 addition & 1 deletion src/include/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
/* #undef PACKAGE */

/* Version number of package */
#define VERSION "2.2.0"
/* #undef VERSION */
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This shouldn't be part of the PR... :)


/* Define for the x86_64 CPU famyly */
/* #undef X86_64 */
18 changes: 16 additions & 2 deletions src/include/fann_activation.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,14 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#define fann_cos_real(sum) (cos(sum)/2.0f+0.5f)
#define fann_cos_derive(steepness, sum) (steepness*-sin(steepness*sum)/2.0f)


/* FANN_RELU */
#define fann_relu_real(sum) (sum > 0.0 ? sum : 0.0)
#define fann_relu_derive(steepness, sum) (sum > 0.0 ? steepness : 0.0)

#define fann_leaky_relu_real(sum) (sum > 0.0 ? sum : (sum / 100.0))
#define fann_leaky_relu_derive(steepness, sum) (sum > 0.0 ? steepness : (steepness / 100.0))

#define fann_activation_switch(activation_function, value, result) \
switch(activation_function) \
{ \
Expand Down Expand Up @@ -137,8 +145,14 @@ switch(activation_function) \
result = (fann_type)fann_cos_real(value); \
break; \
case FANN_GAUSSIAN_STEPWISE: \
result = 0; \
break; \
result = 0; \
break; \
case FANN_RELU: \
result = (fann_type)fann_relu_real(value); \
break; \
case FANN_LEAKY_RELU: \
result = (fann_type)fann_leaky_relu_real(value); \
break; \
}

#endif
8 changes: 6 additions & 2 deletions src/include/fann_data.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,9 @@ enum fann_activationfunc_enum
FANN_SIN_SYMMETRIC,
FANN_COS_SYMMETRIC,
FANN_SIN,
FANN_COS
FANN_COS,
FANN_RELU,
FANN_LEAKY_RELU
};

/* Constant: FANN_ACTIVATIONFUNC_NAMES
Expand Down Expand Up @@ -258,7 +260,9 @@ static char const *const FANN_ACTIVATIONFUNC_NAMES[] = {
"FANN_SIN_SYMMETRIC",
"FANN_COS_SYMMETRIC",
"FANN_SIN",
"FANN_COS"
"FANN_COS",
"FANN_RELU",
"FANN_LEAKY_RELU"
};

/* Enum: fann_errorfunc_enum
Expand Down