diff --git a/src/cnn/creation.c b/src/cnn/creation.c index 9e60c2b..81e418d 100644 --- a/src/cnn/creation.c +++ b/src/cnn/creation.c @@ -140,7 +140,7 @@ void add_convolution(Network* network, int depth_output, int dim_output, int act cnn->d_w[i][j] = (float**)malloc(sizeof(float*)*kernel_size); for (int k=0; k < kernel_size; k++) { cnn->w[i][j][k] = (float*)malloc(sizeof(float)*kernel_size); - cnn->d_w[i][j][k] = (float*)malloc(sizeof(float)*kernel_size); + cnn->d_w[i][j][k] = (float*)calloc(kernel_size, sizeof(float)); } } } @@ -151,19 +151,15 @@ void add_convolution(Network* network, int depth_output, int dim_output, int act cnn->d_bias[i] = (float**)malloc(sizeof(float*)*bias_size); for (int j=0; j < bias_size; j++) { cnn->bias[i][j] = (float*)malloc(sizeof(float)*bias_size); - cnn->d_bias[i][j] = (float*)malloc(sizeof(float)*bias_size); + cnn->d_bias[i][j] = (float*)calloc(bias_size, sizeof(float)); } } + int n_in = network->width[n-1]*network->width[n-1]*network->depth[n-1]; + int n_out = network->width[n]*network->width[n]*network->depth[n]; + initialisation_3d_matrix(network->initialisation, cnn->bias, depth_output, dim_output, dim_output, n_in, n_out); + initialisation_4d_matrix(network->initialisation, cnn->w, depth_input, depth_output, kernel_size, kernel_size, n_in, n_out); create_a_cube_input_layer(network, n, depth_output, bias_size); create_a_cube_input_z_layer(network, n, depth_output, bias_size); - // int n_int = network->width[n-1]*network->width[n-1]*network->depth[n-1]; - // int n_out = network->width[n]*network->width[n]*network->depth[n]; - /* Not currently used - initialisation_3d_matrix(network->initialisation, cnn->bias, depth_output, kernel_size, kernel_size, n_int+n_out); - initialisation_3d_matrix(ZERO, cnn->d_bias, depth_output, kernel_size, kernel_size, n_int+n_out); - initialisation_4d_matrix(network->initialisation, cnn->w, depth_input, depth_output, kernel_size, kernel_size, n_int+n_out); - initialisation_4d_matrix(ZERO, cnn->d_w, depth_input, depth_output, kernel_size, kernel_size, n_int+n_out); - */ network->size++; } @@ -183,21 +179,17 @@ void add_dense(Network* network, int output_units, int activation) { nn->input_units = input_units; nn->output_units = output_units; nn->bias = (float*)malloc(sizeof(float)*output_units); - nn->d_bias = (float*)malloc(sizeof(float)*output_units); + nn->d_bias = (float*)calloc(output_units, sizeof(float)); nn->weights = (float**)malloc(sizeof(float*)*input_units); nn->d_weights = (float**)malloc(sizeof(float*)*input_units); for (int i=0; i < input_units; i++) { nn->weights[i] = (float*)malloc(sizeof(float)*output_units); - nn->d_weights[i] = (float*)malloc(sizeof(float)*output_units); + nn->d_weights[i] = (float*)calloc(output_units, sizeof(float)); } + initialisation_1d_matrix(network->initialisation, nn->bias, output_units, input_units, output_units); + initialisation_2d_matrix(network->initialisation, nn->weights, input_units, output_units, input_units, output_units); create_a_line_input_layer(network, n, output_units); create_a_line_input_z_layer(network, n, output_units); - /* Not currently used - initialisation_1d_matrix(network->initialisation, nn->bias, output_units, output_units+input_units); - initialisation_1d_matrix(ZERO, nn->d_bias, output_units, output_units+input_units); - initialisation_2d_matrix(network->initialisation, nn->weights, input_units, output_units, output_units+input_units); - initialisation_2d_matrix(ZERO, nn->d_weights, input_units, output_units, output_units+input_units); - */ network->size++; } @@ -220,20 +212,16 @@ void add_dense_linearisation(Network* network, int output_units, int activation) nn->output_units = output_units; nn->bias = (float*)malloc(sizeof(float)*output_units); - nn->d_bias = (float*)malloc(sizeof(float)*output_units); + nn->d_bias = (float*)calloc(output_units, sizeof(float)); nn->weights = (float**)malloc(sizeof(float*)*input_units); nn->d_weights = (float**)malloc(sizeof(float*)*input_units); for (int i=0; i < input_units; i++) { nn->weights[i] = (float*)malloc(sizeof(float)*output_units); - nn->d_weights[i] = (float*)malloc(sizeof(float)*output_units); + nn->d_weights[i] = (float*)calloc(output_units, sizeof(float)); } - /* Not currently used - initialisation_1d_matrix(network->initialisation, nn->bias, output_units, output_units+input_units); - initialisation_1d_matrix(ZERO, nn->d_bias, output_units, output_units+input_units); - initialisation_2d_matrix(network->initialisation, nn->weights, input_units, output_units, output_units+input_units); - initialisation_2d_matrix(ZERO, nn->d_weights, input_units, output_units, output_units+input_units); */ + initialisation_1d_matrix(network->initialisation, nn->bias, output_units, input_units, output_units); + initialisation_2d_matrix(network->initialisation, nn->weights, input_units, output_units, input_units, output_units); create_a_line_input_layer(network, n, output_units); create_a_line_input_z_layer(network, n, output_units); - network->size++; } \ No newline at end of file diff --git a/src/cnn/include/initialisation.h b/src/cnn/include/initialisation.h index 837f20d..7eb3c3c 100644 --- a/src/cnn/include/initialisation.h +++ b/src/cnn/include/initialisation.h @@ -5,29 +5,28 @@ #define RAND_FLT() ((float)rand())/((float)RAND_MAX) #define ZERO 0 -#define GLOROT_NORMAL 1 -#define GLOROT_UNIFROM 2 -#define HE_NORMAL 3 -#define HE_UNIFORM 4 +#define GLOROT 1 +#define XAVIER 1 // Xavier and Glorot initialisations are the same +#define HE 2 /* -* Initialise une matrice 1d rows de float en fonction du type d'initialisation +* Initialise une matrice 1d dim de float en fonction du type d'initialisation */ -void initialisation_1d_matrix(int initialisation, float* matrix, int rows, int n); // TODO (UNIFORM AND VARIATIONS) +void initialisation_1d_matrix(int initialisation, float* matrix, int dim, int n_in, int n_out); /* -* Initialise une matrice 2d rows*columns de float en fonction du type d'initialisation +* Initialise une matrice 2d dim1*dim2 de float en fonction du type d'initialisation */ -void initialisation_2d_matrix(int initialisation, float** matrix, int rows, int columns, int n); // TODO +void initialisation_2d_matrix(int initialisation, float** matrix, int dim1, int dim2, int n_in, int n_out); /* -* Initialise une matrice 3d depth*dim*columns de float en fonction du type d'initialisation +* Initialise une matrice 3d depth*dim1*dim2 de float en fonction du type d'initialisation */ -void initialisation_3d_matrix(int initialisation, float*** matrix, int depth, int rows, int columns, int n); // TODO +void initialisation_3d_matrix(int initialisation, float*** matrix, int depth, int dim1, int dim2, int n_in, int n_out); /* -* Initialise une matrice 4d rows*columns*rows1*columns1 de float en fonction du type d'initialisation +* Initialise une matrice 4d depth1*depth2*dim1*dim2 de float en fonction du type d'initialisation */ -void initialisation_4d_matrix(int initialisation, float**** matrix, int rows, int columns, int rows1, int columns1, int n); // TODO +void initialisation_4d_matrix(int initialisation, float**** matrix, int depth1, int depth2, int dim1, int dim2, int n_in, int n_out); #endif \ No newline at end of file diff --git a/src/cnn/initialisation.c b/src/cnn/initialisation.c index 0995122..b479ad8 100644 --- a/src/cnn/initialisation.c +++ b/src/cnn/initialisation.c @@ -4,49 +4,89 @@ #include "../include/colors.h" #include "include/initialisation.h" +// glorot (wavier initialisation) linear, tanh, softmax, logistic (1/(fan_in+fan_out/2)) +// he initialisation : RELU (2/fan_in) +// LeCun initialisation: SELU (1/fan_in) -void initialisation_1d_matrix(int initialisation, float* matrix, int rows, int n) { // TODO - printf_warning("Appel de initialisation_1d_matrix, incomplet\n"); - float lower_bound = -6/sqrt((double)n); - float distance = -lower_bound-lower_bound; - for (int i=0; i < rows; i++) { - matrix[i] = lower_bound + RAND_FLT()*distance; +// Only uniform for the moment +void initialisation_1d_matrix(int initialisation, float* matrix, int dim, int n_in, int n_out) { + int n; + if (initialisation == GLOROT) { + n = (n_in + n_out)/2; + + } else if (initialisation == HE) { + n = n_in/2; + } else { + printf_warning("Initialisation non reconnue dans 'initialisation_1d_matrix' \n"); + return ; + } + float lower_bound = -1/sqrt((double)n); + float distance_bounds = -2*lower_bound; + for (int i=0; i < dim; i++) { + matrix[i] = lower_bound + RAND_FLT()*distance_bounds; } } -void initialisation_2d_matrix(int initialisation, float** matrix, int rows, int columns, int n) { // TODO - printf_warning("Appel de initialisation_2d_matrix, incomplet\n"); - float lower_bound = -6/sqrt((double)n); - float distance = -lower_bound-lower_bound; - for (int i=0; i < rows; i++) { - for (int j=0; j < columns; j++) { - matrix[i][j] = lower_bound + RAND_FLT()*distance; +void initialisation_2d_matrix(int initialisation, float** matrix, int dim1, int dim2, int n_in, int n_out) { + int n; + if (initialisation == GLOROT) { + n = (n_in + n_out)/2; + + } else if (initialisation == HE) { + n = n_in/2; + } else { + printf_warning("Initialisation non reconnue dans 'initialisation_2d_matrix' \n"); + return ; + } + float lower_bound = -1/sqrt((double)n); + float distance_bounds = -2*lower_bound; + for (int i=0; i < dim1; i++) { + for (int j=0; j < dim2; j++) { + matrix[i][j] = lower_bound + RAND_FLT()*distance_bounds; } } } -void initialisation_3d_matrix(int initialisation, float*** matrix, int depth, int rows, int columns, int n) { // TODO - printf_warning("Appel de initialisation_3d_matrix, incomplet\n"); - float lower_bound = -6/sqrt((double)n); - float distance = -lower_bound-lower_bound; +void initialisation_3d_matrix(int initialisation, float*** matrix, int depth, int dim1, int dim2, int n_in, int n_out) { + int n; + if (initialisation == GLOROT) { + n = (n_in + n_out)/2; + + } else if (initialisation == HE) { + n = n_in/2; + } else { + printf_warning("Initialisation non reconnue dans 'initialisation_3d_matrix' \n"); + return ; + } + float lower_bound = -1/sqrt((double)n); + float distance_bounds = -2*lower_bound; for (int i=0; i < depth; i++) { - for (int j=0; j < rows; j++) { - for (int k=0; k < columns; k++) { - matrix[i][j][k] = lower_bound + RAND_FLT()*distance; + for (int j=0; j < dim1; j++) { + for (int k=0; k < dim2; k++) { + matrix[i][j][k] = lower_bound + RAND_FLT()*distance_bounds; } } } } -void initialisation_4d_matrix(int initialisation, float**** matrix, int rows, int columns, int rows1, int columns1, int n) { // TODO - printf_warning("Appel de initialisation_4d_matrix, incomplet\n"); - float lower_bound = -6/sqrt((double)n); - float distance = -lower_bound-lower_bound; - for (int i=0; i < rows; i++) { - for (int j=0; j < columns; j++) { - for (int k=0; k < rows1; k++) { - for (int l=0; l < columns1; l++) { - matrix[i][j][k][l] = lower_bound + RAND_FLT()*distance; +void initialisation_4d_matrix(int initialisation, float**** matrix, int depth1, int depth2, int dim1, int dim2, int n_in, int n_out) { + int n; + if (initialisation == GLOROT) { + n = (n_in + n_out)/2; + + } else if (initialisation == HE) { + n = n_in/2; + } else { + printf_warning("Initialisation non reconnue dans 'initialisation_3d_matrix' \n"); + return ; + } + float lower_bound = -1/sqrt((double)n); + float distance_bounds = -2*lower_bound; + for (int i=0; i < depth1; i++) { + for (int j=0; j < depth2; j++) { + for (int k=0; k < dim1; k++) { + for (int l=0; l < dim2; l++) { + matrix[i][j][k][l] = lower_bound + RAND_FLT()*distance_bounds; } } } diff --git a/src/cnn/main.c b/src/cnn/main.c index 91b6dd0..e9cd7f4 100644 --- a/src/cnn/main.c +++ b/src/cnn/main.c @@ -30,7 +30,7 @@ void help(char* call) { void dev_conv() { - Network* network = create_network_lenet5(0, 0, TANH, GLOROT_NORMAL, 32, 1); + Network* network = create_network_lenet5(0, 0, TANH, GLOROT, 32, 1); forward_propagation(network); } diff --git a/src/cnn/train.c b/src/cnn/train.c index a3b0219..b4aad18 100644 --- a/src/cnn/train.c +++ b/src/cnn/train.c @@ -99,7 +99,7 @@ void train(int dataset_type, char* images_file, char* labels_file, char* data_di } // Initialisation du réseau - Network* network = create_network_lenet5(0, 0, TANH, GLOROT_NORMAL, input_dim, input_depth); + Network* network = create_network_lenet5(0, 0, TANH, GLOROT, input_dim, input_depth); #ifdef USE_MULTITHREADING // Récupération du nombre de threads disponibles